latinfo 0.11.0 → 0.12.0

package/dist/index.js

@@ -47,7 +47,7 @@ const local_search_1 = require("./local-search");
 const client_search_1 = require("./client-search");
 const odis_search_1 = require("./odis-search");
 const mphf_search_1 = require("./mphf-search");
-const VERSION = '0.10.0';
+const VERSION = '0.11.0';
 const API_URL = process.env.LATINFO_API_URL || 'https://api.latinfo.dev';
 const GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID || 'Ov23li5fcQaiCsVtaMKK';
 const CONFIG_DIR = path_1.default.join(os_1.default.homedir(), '.latinfo');
@@ -1791,33 +1791,43 @@ async function pipeTest(args) {
     const useEasypipe = !fs_1.default.existsSync(scriptPath);
     const cmd = useEasypipe
         ? `npx tsx ${easypipePath} ${yamlPath} --limit 100 --local`
-        : `npx tsx ${scriptPath} --limit 100`;
+        : `npx tsx ${scriptPath} --limit 100 --local`;
     console.log(`[pipe] Gate 1: TEST (100 records)\n`);
     console.log(`Running: ${cmd}\n`);
+    let output = '';
     try {
-        run(cmd, { stdio: 'inherit', cwd: repo });
+        output = run(cmd, { encoding: 'utf-8', cwd: repo, stdio: ['inherit', 'pipe', 'inherit'] }) || '';
+        // Print output
+        if (output)
+            process.stdout.write(output);
     }
-    catch {
+    catch (e) {
+        if (e.stdout) {
+            output = e.stdout;
+            process.stdout.write(output);
+        }
         errors.push('Import script failed');
     }
-    // Validate output files exist
-    const outDir = `/tmp/${sourceName}-import`;
+    // Validate from script output (files are cleaned up by script)
     if (errors.length === 0) {
-        const binFiles = fs_1.default.readdirSync(outDir || '/tmp').filter(f => f.startsWith(sourceName) && f.endsWith('.bin'));
-        if (binFiles.length === 0)
-            errors.push('No .bin files generated');
-        const idxFile = path_1.default.join(outDir, `${sourceName}.idx`);
-        if (!fs_1.default.existsSync(idxFile) && !fs_1.default.existsSync(`/tmp/${sourceName}.idx`)) {
-            // Check R2 upload happened (for non-local mode)
+        if (!output.includes('Success') && !output.includes('records')) {
+            errors.push('Import did not report success');
+        }
+        // Check for V2 search index (not V1)
+        if (output.includes('V1)') && !output.includes('V2')) {
+            errors.push('Search index is V1 — MUST use V2. Add statusFieldIndex to buildSearchIndex()');
         }
-        // Check for V2 search index
-        const searchIdx = fs_1.default.readdirSync('/tmp').filter(f => f.includes(sourceName) && f.includes('search.idx'));
-        if (searchIdx.length === 0)
-            errors.push('No V2 search index generated — use statusFieldIndex in buildSearchIndex');
         // Check for MPHF
-        const mphf = fs_1.default.readdirSync('/tmp').filter(f => f.includes(sourceName) && f.includes('.mphf'));
-        if (mphf.length === 0)
-            errors.push('No MPHF generated — call buildMphfFromIdx after buildSearchIndex');
+        if (!output.includes('[mphf]') && !output.includes('MPHF')) {
+            errors.push('No MPHF generated — call buildMphfFromIdx() after buildSearchIndex()');
+        }
+        // Check record count
+        const recordMatch = output.match(/(\d[\d,]*)\s*records/);
+        if (recordMatch) {
+            const count = parseInt(recordMatch[1].replace(/,/g, ''));
+            if (count === 0)
+                errors.push('Zero records imported');
+        }
     }
     if (errors.length > 0) {
         console.error(`\n[pipe] Gate 1 FAILED:`);
@@ -1847,8 +1857,8 @@ async function pipeValidate(args) {
     const useEasypipe = !fs_1.default.existsSync(scriptPath);
     const cmd = useEasypipe
         ? `npx tsx ${easypipePath} ${path_1.default.join(repo, 'sources', `${sourceName}.yaml`)} --local`
-        : `npx tsx ${scriptPath}`;
-    console.log(`[pipe] Gate 2: VALIDATE (full import)\n`);
+        : `npx tsx ${scriptPath} --local`;
+    console.log(`[pipe] Gate 2: VALIDATE (full import, local only — no R2 upload)\n`);
     console.log(`Running: ${cmd}\n`);
     try {
         const output = run(cmd, { cwd: repo, stdio: 'inherit', encoding: 'utf-8' });
@@ -1875,21 +1885,39 @@ async function pipeStage(args) {
     const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
     const RUNNER = 'f3mt0@100.109.82.87';
     console.log(`[pipe] Gate 3: STAGE (Linux Mint — import + bench)\n`);
-    // 1. Copy script + YAML to runner
+    // 1. Copy script + YAML to runner via scp
     const repo = getRepoPath();
-    console.log('[pipe] Syncing repo on Linux Mint...');
+    const remoteRepo = '~/actions-runner/_work/latinfo-api/latinfo-api';
+    console.log('[pipe] Syncing files to Linux Mint...');
     try {
-        run(`ssh ${RUNNER} "cd ~/actions-runner/_work/latinfo-api/latinfo-api && git pull"`, { stdio: 'inherit' });
+        run(`ssh ${RUNNER} "echo OK"`, { stdio: 'pipe', timeout: 10_000 });
+        // Copy import script and YAML
+        const scriptFile = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
+        const yamlFile = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
+        if (fs_1.default.existsSync(scriptFile))
+            run(`scp ${scriptFile} ${RUNNER}:${remoteRepo}/src/imports/`, { stdio: 'pipe' });
+        if (fs_1.default.existsSync(yamlFile))
+            run(`scp ${yamlFile} ${RUNNER}:${remoteRepo}/sources/`, { stdio: 'pipe' });
+        console.log('[pipe] Files synced.');
     }
     catch {
         console.error('[pipe] SSH failed. Is Linux Mint running? Check: ssh f3mt0@100.109.82.87');
         process.exit(1);
     }
-    // 2. Run import on Linux Mint
-    const scriptPath = `src/imports/${sourceName}.ts`;
+    // 2. Sync env files if missing on runner
+    try {
+        const envLocal = path_1.default.join(repo, '.env');
+        const devVarsLocal = path_1.default.join(repo, '.dev.vars');
+        if (fs_1.default.existsSync(envLocal))
+            run(`scp ${envLocal} ${RUNNER}:${remoteRepo}/.env`, { stdio: 'pipe' });
+        if (fs_1.default.existsSync(devVarsLocal))
+            run(`scp ${devVarsLocal} ${RUNNER}:${remoteRepo}/.dev.vars`, { stdio: 'pipe' });
+    }
+    catch { }
+    // 3. Run import on Linux Mint
     console.log(`[pipe] Running import on Linux Mint...`);
     try {
-        run(`ssh ${RUNNER} "cd ~/actions-runner/_work/latinfo-api/latinfo-api && npx tsx ${scriptPath}"`, {
+        run(`ssh ${RUNNER} "cd ${remoteRepo} && set -a && source .env 2>/dev/null; source .dev.vars 2>/dev/null; set +a && R2_BUCKET_NAME=latinfo-data npx tsx src/imports/${sourceName}.ts"`, {
             stdio: 'inherit', timeout: 600_000,
         });
     }
@@ -1942,6 +1970,147 @@ p50:lats[Math.floor(lats.length*0.5)],p95:lats[Math.floor(lats.length*0.95)],p99
         savePipeStatus(status);
     }
 }
+async function pipeDocs(args) {
+    const [sourceName, docPath] = args;
+    if (!sourceName) {
+        console.error(`Usage: latinfo pipe docs <source-name> [doc-file]
+
+If no doc-file is provided, generates a template for you to fill in.
+If doc-file is provided, copies it as the source documentation.
+
+The documentation MUST include these sections:
+
+  ## Source
+  URL, institution, what data it contains, update frequency
+
+  ## How it works
+  Download method (fetch, Playwright, API), authentication, CAPTCHA, encoding
+
+  ## Fields
+  All fields with types and examples
+
+  ## Known issues
+  Encoding problems, rate limits, CAPTCHA changes, session handling
+
+  ## Troubleshooting
+  What to do if:
+  - URL changes
+  - CAPTCHA type changes
+  - Encoding changes
+  - API response format changes
+  - Authentication method changes
+  - Rate limits increase
+  - Data format changes (new columns, removed columns)
+
+  ## Dependencies
+  Required packages and why (playwright, ddddocr, etc.)
+
+  ## Bench results
+  Concurrent users tested, success rate, p50/p95/p99`);
+        process.exit(1);
+    }
+    const status = loadPipeStatus(sourceName);
+    requireGate(status, 'stage', 'docs');
+    const repo = getRepoPath();
+    const docsDir = path_1.default.join(repo, 'docs', 'sources');
+    fs_1.default.mkdirSync(docsDir, { recursive: true });
+    const destPath = path_1.default.join(docsDir, `${sourceName}.md`);
+    if (docPath) {
+        // Copy provided doc
+        const src = path_1.default.resolve(docPath);
+        if (!fs_1.default.existsSync(src)) {
+            console.error(`File not found: ${src}`);
+            process.exit(1);
+        }
+        const content = fs_1.default.readFileSync(src, 'utf-8');
+        // Validate required sections
+        const required = ['## Source', '## How it works', '## Fields', '## Known issues', '## Troubleshooting'];
+        const missing = required.filter(s => !content.includes(s));
+        if (missing.length > 0) {
+            console.error(`[pipe] Documentation missing required sections:`);
+            for (const m of missing)
+                console.error(`  ✗ ${m}`);
+            console.error(`\nAdd these sections to your doc and try again.`);
+            process.exit(1);
+        }
+        fs_1.default.copyFileSync(src, destPath);
+        console.log(`[pipe] Documentation saved: ${destPath}`);
+    }
+    else {
+        // Generate template
+        const yamlPath = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
+        const yamlContent = fs_1.default.existsSync(yamlPath) ? fs_1.default.readFileSync(yamlPath, 'utf-8') : '';
+        const urlMatch = yamlContent.match(/url:\s*(.+)/);
+        const url = urlMatch ? urlMatch[1].trim() : 'https://example.com';
+        const benchData = status.stage?.bench;
+        const template = `# ${sourceName}
+
+## Source
+- **URL**: ${url}
+- **Institution**: TODO
+- **Data**: TODO (what records this contains)
+- **Records**: TODO (approximate count)
+- **Update frequency**: TODO (daily, weekly, manual)
+- **Format**: TODO (CSV, JSON API, web scraping)
+
+## How it works
+TODO: Describe step by step how the import script works.
+- How is data downloaded? (direct URL, API with pagination, Playwright crawler)
+- Is there authentication? (API key, session, CAPTCHA)
+- What encoding is the source? (UTF-8, ISO-8859-1)
+- Any special parsing needed? (date formats, amount formats, field concatenation)
+
+## Fields
+| Field | Type | Example | Notes |
+|-------|------|---------|-------|
+| TODO | string | TODO | TODO |
+
+## Known issues
+- TODO: List any encoding problems, edge cases, data quality issues
+- TODO: Rate limits, CAPTCHA difficulty, session expiration
+
+## Troubleshooting
+
+### URL changes
+TODO: Where to find the new URL, how to update
+
+### CAPTCHA changes
+TODO: What CAPTCHA solver is used, alternatives if it breaks
+
+### Encoding changes
+TODO: Current encoding, how to detect changes
+
+### Format changes
+TODO: How to detect if columns change, new fields added, fields removed
+
+### Authentication changes
+TODO: Current auth method, what to check if it stops working
+
+## Dependencies
+TODO: List npm packages and why each is needed
+\`\`\`
+playwright — browser automation for CAPTCHA/session
+ddddocr — CAPTCHA OCR solver
+\`\`\`
+
+## Bench results
+${benchData ? `- **Concurrent**: ${benchData.concurrent}
+- **Success rate**: ${benchData.success_rate.toFixed(1)}%
+- **p50**: ${benchData.p50}ms
+- **p95**: ${benchData.p95}ms
+- **p99**: ${benchData.p99}ms` : 'TODO: Run latinfo pipe stage first'}
+`;
+        fs_1.default.writeFileSync(destPath, template);
+        console.log(`[pipe] Template generated: ${destPath}`);
+        console.log(`\nFill in the TODO sections, then run:`);
+        console.log(`  latinfo pipe docs ${sourceName} ${destPath}`);
+        process.exit(1); // Force them to fill it in
+    }
+    console.log(`\n[pipe] Gate 3.5 PASSED ✓`);
+    console.log(`[pipe] Next: latinfo pipe publish ${sourceName}`);
+    status.docs = { passed: true, timestamp: new Date().toISOString() };
+    savePipeStatus(status);
+}
 async function pipePublish(args) {
     const [sourceName] = args;
     if (!sourceName) {
@@ -1952,6 +2121,11 @@ async function pipePublish(args) {
     requireGate(status, 'test', 'publish');
     requireGate(status, 'validate', 'publish');
     requireGate(status, 'stage', 'publish');
+    if (!status.docs?.passed) {
+        console.error(`[pipe] Gate "docs" has not passed. Run: latinfo pipe docs ${sourceName}`);
+        console.error(`[pipe] Documentation is required before publishing.`);
+        process.exit(1);
+    }
     const repo = getRepoPath();
     const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
     const RUNNER = 'f3mt0@100.109.82.87';
@@ -1963,6 +2137,9 @@ async function pipePublish(args) {
     if (fs_1.default.existsSync(scriptPath))
         files.push(`src/imports/${sourceName}.ts`);
     try {
+        const docsFile = `docs/sources/${sourceName}.md`;
+        if (fs_1.default.existsSync(path_1.default.join(repo, docsFile)))
+            files.push(docsFile);
         run(`git add ${files.join(' ')} src/sources.ts .github/workflows/import.yml`, { cwd: repo, stdio: 'pipe' });
         run(`git commit -m "Add data source: ${sourceName}"`, { cwd: repo, stdio: 'pipe' });
         run(`git push`, { cwd: repo, stdio: 'pipe' });
@@ -2007,7 +2184,7 @@ async function pipeStatus(args) {
     const [sourceName] = args;
     if (sourceName) {
         const status = loadPipeStatus(sourceName);
-        const gates = ['test', 'validate', 'stage', 'publish'];
+        const gates = ['test', 'validate', 'stage', 'docs', 'publish'];
         console.log(`Source: ${sourceName}\n`);
         for (const gate of gates) {
             const g = status[gate];
@@ -2033,7 +2210,7 @@ async function pipeStatus(args) {
         const files = fs_1.default.readdirSync(PIPE_STATUS_DIR).filter(f => f.endsWith('.json'));
         for (const f of files) {
             const s = JSON.parse(fs_1.default.readFileSync(path_1.default.join(PIPE_STATUS_DIR, f), 'utf-8'));
-            const gates = ['test', 'validate', 'stage', 'publish'];
+            const gates = ['test', 'validate', 'stage', 'docs', 'publish'];
             const icons = gates.map(g => s[g]?.passed ? '✓' : s[g] ? '✗' : '⬚').join('');
             console.log(`  ${s.source}  [${icons}]`);
         }
@@ -2061,6 +2238,9 @@ async function pipe(args) {
         case 'stage':
             await pipeStage(subArgs);
             break;
+        case 'docs':
+            await pipeDocs(subArgs);
+            break;
         case 'publish':
             await pipePublish(subArgs);
             break;
@@ -2096,6 +2276,7 @@ COMMANDS
   test <source>                                      Gate 1: test 100 records locally
   validate <source>                                  Gate 2: full import locally
   stage <source>                                     Gate 3: import + 500 bench on Linux Mint
+  docs <source> [doc-file]                           Gate 3.5: write/upload documentation
   publish <source>                                   Gate 4: deploy to production
   status [source]                                    Show gate status
   list                                               List all sources
@@ -2105,6 +2286,7 @@ GATES (each must pass before the next unlocks)
   test     → 100 records, validates IDs, encoding, V2 search, MPHF
   validate → full import, all records, field validation
   stage    → Linux Mint: import + 500 concurrent bench (99.9% required)
+  docs     → documentation with required sections (Source, How it works, Fields, etc.)
   publish  → production: deploy + smoke test + bench + rollback on failure
 
 WORKFLOW

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "latinfo",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "Tax registry & procurement API for Latin America. Query RUC, DNI, NIT, licitaciones from Peru & Colombia. Offline MPHF search, full OCDS data, updated daily.",
   "homepage": "https://latinfo.dev",
   "repository": {