latinfo 0.15.0 → 0.16.0

package/dist/index.js

@@ -47,7 +47,7 @@ const local_search_1 = require("./local-search");
 const client_search_1 = require("./client-search");
 const odis_search_1 = require("./odis-search");
 const mphf_search_1 = require("./mphf-search");
-const VERSION = '0.15.0';
+const VERSION = '0.16.0';
 const API_URL = process.env.LATINFO_API_URL || 'https://api.latinfo.dev';
 const GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID || 'Ov23li5fcQaiCsVtaMKK';
 const CONFIG_DIR = path_1.default.join(os_1.default.homedir(), '.latinfo');
@@ -2041,15 +2041,15 @@ async function pipeStage(args) {
             }
         }
     }
-    // 3b. Install Python deps on Linux Mint (using venv)
+    // 3b. Install Python deps on Linux Mint (try multiple methods)
     const pyDepsMatch = yamlContent.match(/python_dependencies:\n([\s\S]*?)(?=\n\w|\n$|$)/);
     if (pyDepsMatch) {
         const pyDeps = pyDepsMatch[1].split('\n').map(l => l.replace(/^\s*-\s*/, '').trim()).filter(Boolean);
         if (pyDeps.length > 0) {
+            const pkgs = pyDeps.join(' ');
             console.log(`[pipe] Installing Python deps on Linux Mint: ${pyDeps.join(', ')}...`);
             try {
-                // Create venv if not exists, activate, install
-                run(`ssh ${RUNNER} "cd ${remoteRepo} && (test -d .venv || python3 -m venv .venv) && .venv/bin/pip install ${pyDeps.join(' ')}"`, { stdio: 'inherit' });
+                run(`ssh ${RUNNER} "python3 -m pip install --break-system-packages ${pkgs} 2>/dev/null || pip3 install --break-system-packages ${pkgs} 2>/dev/null || python3 -m pip install --user ${pkgs} 2>/dev/null || pip3 install ${pkgs}"`, { stdio: 'inherit' });
             }
             catch {
                 console.error('[pipe] Failed to install Python deps on Linux Mint');
@@ -2060,7 +2060,7 @@ async function pipeStage(args) {
     // 4. Run import on Linux Mint
     console.log(`[pipe] Running import on Linux Mint...`);
     try {
-        run(`ssh ${RUNNER} "cd ${remoteRepo} && set -a && source .env 2>/dev/null; source .dev.vars 2>/dev/null; test -f .venv/bin/activate && source .venv/bin/activate; set +a && R2_BUCKET_NAME=latinfo-data npx tsx src/imports/${sourceName}.ts"`, {
+        run(`ssh ${RUNNER} "cd ${remoteRepo} && set -a && source .env 2>/dev/null; source .dev.vars 2>/dev/null; set +a && R2_BUCKET_NAME=latinfo-data npx tsx src/imports/${sourceName}.ts"`, {
             stdio: 'inherit',
         });
     }
@@ -2282,9 +2282,41 @@ async function pipePublish(args) {
         console.error(`[pipe] Failed to generate sources.ts`);
         process.exit(1);
     }
-    // 2. Git add + commit + push
+    // 2. Auto-update import.yml matrix
+    console.log(`[pipe] Updating import.yml...`);
+    const importYmlPath = path_1.default.join(repo, '.github', 'workflows', 'import.yml');
+    if (fs_1.default.existsSync(importYmlPath)) {
+        let yml = fs_1.default.readFileSync(importYmlPath, 'utf-8');
+        // Add to options list if not present
+        if (!yml.includes(`- ${sourceName}`)) {
+            yml = yml.replace(/( +- all\n)/, `          - ${sourceName}\n$1`);
+        }
+        // Add to matrix includes if not present
+        if (!yml.includes(`source: ${sourceName}`)) {
+            const yamlContent = fs_1.default.readFileSync(path_1.default.join(repo, 'sources', `${sourceName}.yaml`), 'utf-8');
+            const importScriptMatch = yamlContent.match(/import_script:\s*(.+)/);
+            const script = importScriptMatch ? importScriptMatch[1].trim() : `src/imports/${sourceName}.ts`;
+            const schedule = yamlContent.includes('schedule: daily') || yamlContent.includes('schedule: weekly');
+            const needsBrowser = yamlContent.includes('playwright');
+            let matrixEntry = `\n          - source: ${sourceName}\n`;
+            matrixEntry += `            script: ${script}\n`;
+            matrixEntry += `            timeout: 15\n`;
+            matrixEntry += `            daily: ${schedule}\n`;
+            if (needsBrowser)
+                matrixEntry += `            needs_browser: true\n`;
+            matrixEntry += `            smoke_url: ""\n`;
+            matrixEntry += `            smoke_field: ""\n`;
+            // Insert before co-rues-registry (last entry) or at end of matrix
+            const coRuesMatch = yml.match(/(\n +- source: co-rues-registry)/);
+            if (coRuesMatch && coRuesMatch.index) {
+                yml = yml.slice(0, coRuesMatch.index) + matrixEntry + yml.slice(coRuesMatch.index);
+            }
+        }
+        fs_1.default.writeFileSync(importYmlPath, yml);
+    }
+    // 3. Git add + commit + push
     console.log(`[pipe] Committing to repo...`);
-    const files = [`sources/${sourceName}.yaml`, 'src/sources.ts'];
+    const files = [`sources/${sourceName}.yaml`, 'src/sources.ts', '.github/workflows/import.yml'];
     const scriptPath = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
     const yamlContent = fs_1.default.readFileSync(path_1.default.join(repo, 'sources', `${sourceName}.yaml`), 'utf-8');
     const importScriptMatch = yamlContent.match(/import_script:\s*(.+)/);
@@ -2473,6 +2505,10 @@ NAMING
   {country}-{institution}-{dataset}, all lowercase english.
   Examples: pe-sunat-padron, pe-osce-sanctioned, co-rues-registry
 
+LANGUAGE
+  All code, documentation, field names, dataset names, and commit messages
+  must be in English. No Spanish, Portuguese, or other languages.
+
 ENVIRONMENT
   LATINFO_ADMIN_SECRET  Auto-detected from ~/.latinfo/admin.secret or .dev.vars
   LATINFO_REPO_PATH     Auto-detected from cwd`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "latinfo",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "description": "Tax registry & procurement API for Latin America. Query RUC, DNI, NIT, licitaciones from Peru & Colombia. Offline MPHF search, full OCDS data, updated daily.",
   "homepage": "https://latinfo.dev",
   "repository": {