latinfo 0.13.2 → 0.14.0

package/dist/index.js

@@ -47,7 +47,7 @@ const local_search_1 = require("./local-search");
 const client_search_1 = require("./client-search");
 const odis_search_1 = require("./odis-search");
 const mphf_search_1 = require("./mphf-search");
-const VERSION = '0.13.2';
+const VERSION = '0.14.0';
 const API_URL = process.env.LATINFO_API_URL || 'https://api.latinfo.dev';
 const GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID || 'Ov23li5fcQaiCsVtaMKK';
 const CONFIG_DIR = path_1.default.join(os_1.default.homedir(), '.latinfo');
@@ -1724,9 +1724,16 @@ async function pipeScript(args) {
     console.log(`[pipe] Gates reset — run: latinfo pipe test ${sourceName}`);
 }
 async function pipeDeps(args) {
-    const [sourceName, ...deps] = args;
+    const isPython = args.includes('--python');
+    const filtered = args.filter(a => a !== '--python');
+    const [sourceName, ...deps] = filtered;
     if (!sourceName || deps.length === 0) {
-        console.error('Usage: latinfo pipe deps <source-name> <pkg1> [pkg2] ...');
+        console.error(`Usage: latinfo pipe deps <source-name> <pkg1> [pkg2] ...
+       latinfo pipe deps <source-name> --python <pkg1> [pkg2] ...
+
+Examples:
+  latinfo pipe deps pe-redam-registry playwright          # npm package
+  latinfo pipe deps pe-redam-registry --python ddddocr    # Python package (pip3)`);
         process.exit(1);
     }
     const repo = getRepoPath();
@@ -1735,26 +1742,37 @@ async function pipeDeps(args) {
         console.error(`Source not found: ${yamlPath}`);
         process.exit(1);
     }
-    // Add dependencies to YAML
     let yaml = fs_1.default.readFileSync(yamlPath, 'utf-8');
-    if (yaml.includes('dependencies:')) {
-        // Replace existing deps
-        yaml = yaml.replace(/dependencies:[\s\S]*?(?=\n\w|\n$|$)/, `dependencies:\n${deps.map(d => `  - ${d}`).join('\n')}\n`);
+    const key = isPython ? 'python_dependencies' : 'dependencies';
+    if (yaml.includes(`${key}:`)) {
+        yaml = yaml.replace(new RegExp(`${key}:[\\s\\S]*?(?=\\n\\w|\\n$|$)`), `${key}:\n${deps.map(d => `  - ${d}`).join('\n')}\n`);
     }
     else {
-        yaml += `\ndependencies:\n${deps.map(d => `  - ${d}`).join('\n')}\n`;
+        yaml += `\n${key}:\n${deps.map(d => `  - ${d}`).join('\n')}\n`;
     }
     fs_1.default.writeFileSync(yamlPath, yaml);
-    // Install deps in repo
-    console.log(`[pipe] Installing: ${deps.join(', ')}...`);
     const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
-    try {
-        run(`npm install ${deps.join(' ')}`, { cwd: repo, stdio: 'inherit' });
-        console.log(`[pipe] Dependencies installed and added to YAML.`);
+    if (isPython) {
+        console.log(`[pipe] Installing Python: ${deps.join(', ')}...`);
+        try {
+            run(`pip3 install ${deps.join(' ')}`, { cwd: repo, stdio: 'inherit' });
+            console.log(`[pipe] Python dependencies installed and added to YAML.`);
+        }
+        catch {
+            console.error(`[pipe] Failed to install Python dependencies.`);
+            process.exit(1);
+        }
     }
-    catch {
-        console.error(`[pipe] Failed to install dependencies.`);
-        process.exit(1);
+    else {
+        console.log(`[pipe] Installing npm: ${deps.join(', ')}...`);
+        try {
+            run(`npm install ${deps.join(' ')}`, { cwd: repo, stdio: 'inherit' });
+            console.log(`[pipe] npm dependencies installed and added to YAML.`);
+        }
+        catch {
+            console.error(`[pipe] Failed to install npm dependencies.`);
+            process.exit(1);
+        }
     }
 }
 async function pipeTest(args) {
@@ -1774,17 +1792,28 @@ async function pipeTest(args) {
     const errors = [];
     // Install deps from YAML if present
     const yamlContent = fs_1.default.readFileSync(yamlPath, 'utf-8');
-    const depsMatch = yamlContent.match(/dependencies:\n([\s\S]*?)(?=\n\w|\n$|$)/);
+    const depsMatch = yamlContent.match(/^dependencies:\n([\s\S]*?)(?=\n\w|\n$|$)/m);
     if (depsMatch) {
         const deps = depsMatch[1].split('\n').map(l => l.replace(/^\s*-\s*/, '').trim()).filter(Boolean);
         if (deps.length > 0) {
-            console.log(`[pipe] Installing dependencies: ${deps.join(', ')}...`);
+            console.log(`[pipe] Installing npm deps: ${deps.join(', ')}...`);
             try {
                 run(`npm install ${deps.join(' ')}`, { cwd: repo, stdio: 'pipe' });
             }
             catch { }
         }
     }
+    const pyDepsMatch = yamlContent.match(/python_dependencies:\n([\s\S]*?)(?=\n\w|\n$|$)/);
+    if (pyDepsMatch) {
+        const pyDeps = pyDepsMatch[1].split('\n').map(l => l.replace(/^\s*-\s*/, '').trim()).filter(Boolean);
+        if (pyDeps.length > 0) {
+            console.log(`[pipe] Installing Python deps: ${pyDeps.join(', ')}...`);
+            try {
+                run(`pip3 install ${pyDeps.join(' ')}`, { cwd: repo, stdio: 'pipe' });
+            }
+            catch { }
+        }
+    }
     // Run import with --limit 100
     // Check YAML for custom import_script path
     const importScriptMatch = yamlContent.match(/import_script:\s*(.+)/);
@@ -1899,7 +1928,7 @@ async function pipeStage(args) {
     const remoteRepo = '~/actions-runner/_work/latinfo-api/latinfo-api';
     console.log('[pipe] Syncing files to Linux Mint...');
     try {
-        run(`ssh ${RUNNER} "echo OK"`, { stdio: 'pipe', timeout: 10_000 });
+        run(`ssh -o ConnectTimeout=10 ${RUNNER} "echo OK"`, { stdio: 'pipe' });
         // Copy import script and YAML
         const scriptFile = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
         const yamlFile = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
@@ -1932,11 +1961,11 @@ async function pipeStage(args) {
         if (deps.length > 0) {
             console.log(`[pipe] Installing deps on Linux Mint: ${deps.join(', ')}...`);
             try {
-                run(`ssh ${RUNNER} "cd ${remoteRepo} && npm install ${deps.join(' ')}"`, { stdio: 'inherit', timeout: 120_000 });
+                run(`ssh ${RUNNER} "cd ${remoteRepo} && npm install ${deps.join(' ')}"`, { stdio: 'inherit' });
                 // Install playwright browsers if needed
                 if (deps.some(d => d.includes('playwright'))) {
-                    console.log(`[pipe] Installing Playwright browser...`);
-                    run(`ssh ${RUNNER} "cd ${remoteRepo} && npx playwright install chromium --with-deps"`, { stdio: 'inherit', timeout: 120_000 });
+                    console.log(`[pipe] Installing Playwright browser (this may take a few minutes)...`);
+                    run(`ssh ${RUNNER} "cd ${remoteRepo} && npx playwright install chromium --with-deps"`, { stdio: 'inherit' });
                 }
             }
             catch {
@@ -1945,11 +1974,26 @@ async function pipeStage(args) {
             }
         }
     }
+    // 3b. Install Python deps on Linux Mint
+    const pyDepsMatch = yamlContent.match(/python_dependencies:\n([\s\S]*?)(?=\n\w|\n$|$)/);
+    if (pyDepsMatch) {
+        const pyDeps = pyDepsMatch[1].split('\n').map(l => l.replace(/^\s*-\s*/, '').trim()).filter(Boolean);
+        if (pyDeps.length > 0) {
+            console.log(`[pipe] Installing Python deps on Linux Mint: ${pyDeps.join(', ')}...`);
+            try {
+                run(`ssh ${RUNNER} "pip3 install ${pyDeps.join(' ')}"`, { stdio: 'inherit' });
+            }
+            catch {
+                console.error('[pipe] Failed to install Python deps on Linux Mint');
+                process.exit(1);
+            }
+        }
+    }
     // 4. Run import on Linux Mint
     console.log(`[pipe] Running import on Linux Mint...`);
     try {
         run(`ssh ${RUNNER} "cd ${remoteRepo} && set -a && source .env 2>/dev/null; source .dev.vars 2>/dev/null; set +a && R2_BUCKET_NAME=latinfo-data npx tsx src/imports/${sourceName}.ts"`, {
-            stdio: 'inherit', timeout: 600_000,
+            stdio: 'inherit',
         });
     }
     catch {
@@ -1977,7 +2021,7 @@ Promise.all(Array.from({length:CONC},()=>go())).then(()=>{
 lats.sort((a,b)=>a-b);
 console.log(JSON.stringify({total_ms:Date.now()-t0,success,fails,qps:Math.round(TOTAL/((Date.now()-t0)/1000)),
 p50:lats[Math.floor(lats.length*0.5)],p95:lats[Math.floor(lats.length*0.95)],p99:lats[Math.floor(lats.length*0.99)]}));
-});\\"" `, { encoding: 'utf-8', stdio: 'pipe', timeout: 60_000 });
+});\\"" `, { encoding: 'utf-8', stdio: 'pipe' });
         const bench = JSON.parse(benchOutput.trim());
         const successRate = ((bench.success) / 500 * 100);
         console.log(`\n  500 concurrent: ${bench.qps} q/s, ${successRate.toFixed(1)}% success`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "latinfo",
-  "version": "0.13.2",
+  "version": "0.14.0",
   "description": "Tax registry & procurement API for Latin America. Query RUC, DNI, NIT, licitaciones from Peru & Colombia. Offline MPHF search, full OCDS data, updated daily.",
   "homepage": "https://latinfo.dev",
   "repository": {