latinfo 0.14.0 → 0.15.0

package/dist/index.js

@@ -47,7 +47,7 @@ const local_search_1 = require("./local-search");
 const client_search_1 = require("./client-search");
 const odis_search_1 = require("./odis-search");
 const mphf_search_1 = require("./mphf-search");
-const VERSION = '0.14.0';
+const VERSION = '0.15.0';
 const API_URL = process.env.LATINFO_API_URL || 'https://api.latinfo.dev';
 const GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID || 'Ov23li5fcQaiCsVtaMKK';
 const CONFIG_DIR = path_1.default.join(os_1.default.homedir(), '.latinfo');
@@ -1724,16 +1724,31 @@ async function pipeScript(args) {
     console.log(`[pipe] Gates reset — run: latinfo pipe test ${sourceName}`);
 }
 async function pipeDeps(args) {
+    // Parse flags
     const isPython = args.includes('--python');
-    const filtered = args.filter(a => a !== '--python');
+    const isRemove = args.includes('--remove');
+    const isList = args.includes('--list');
+    const filtered = args.filter(a => !a.startsWith('--'));
     const [sourceName, ...deps] = filtered;
-    if (!sourceName || deps.length === 0) {
-        console.error(`Usage: latinfo pipe deps <source-name> <pkg1> [pkg2] ...
-       latinfo pipe deps <source-name> --python <pkg1> [pkg2] ...
+    if (!sourceName) {
+        console.error(`Usage: latinfo pipe deps <source> <pkg[@version]> [pkg2] ...   Add npm deps
+       latinfo pipe deps <source> --python <pkg[==version]> [pkg2]    Add Python deps
+       latinfo pipe deps <source> --list                              Show all deps
+       latinfo pipe deps <source> --remove <pkg>                      Remove a dep
+       latinfo pipe deps <source> --remove --python <pkg>             Remove a Python dep
 
 Examples:
-  latinfo pipe deps pe-redam-registry playwright          # npm package
-  latinfo pipe deps pe-redam-registry --python ddddocr    # Python package (pip3)`);
+  latinfo pipe deps pe-source playwright                   # npm, latest
+  latinfo pipe deps pe-source cheerio@1.0.0                # npm, specific version
+  latinfo pipe deps pe-source --python ddddocr             # Python, latest
+  latinfo pipe deps pe-source --python ddddocr==1.4.11     # Python, specific version
+  latinfo pipe deps pe-source --python requests numpy      # multiple Python deps
+  latinfo pipe deps pe-source --list                       # show all deps
+  latinfo pipe deps pe-source --remove cheerio             # remove npm dep
+  latinfo pipe deps pe-source --remove --python ddddocr    # remove Python dep
+
+npm deps install locally + on Linux Mint during stage.
+Python deps only install on Linux Mint during stage (not locally).`);
         process.exit(1);
     }
     const repo = getRepoPath();
@@ -1743,31 +1758,82 @@ Examples:
         process.exit(1);
     }
     let yaml = fs_1.default.readFileSync(yamlPath, 'utf-8');
+    // Helper: read deps list from YAML
+    const readDeps = (key) => {
+        const match = yaml.match(new RegExp(`^${key}:\\n([\\s\\S]*?)(?=\\n\\w|\\n$|$)`, 'm'));
+        if (!match)
+            return [];
+        return match[1].split('\n').map(l => l.replace(/^\s*-\s*/, '').trim()).filter(Boolean);
+    };
+    // Helper: write deps list to YAML
+    const writeDeps = (key, list) => {
+        const block = list.length > 0 ? `${key}:\n${list.map(d => `  - ${d}`).join('\n')}\n` : '';
+        if (yaml.includes(`${key}:`)) {
+            yaml = yaml.replace(new RegExp(`${key}:[\\s\\S]*?(?=\\n\\w|\\n$|$)`), block);
+        }
+        else if (block) {
+            yaml += `\n${block}`;
+        }
+        fs_1.default.writeFileSync(yamlPath, yaml);
+    };
     const key = isPython ? 'python_dependencies' : 'dependencies';
-    if (yaml.includes(`${key}:`)) {
-        yaml = yaml.replace(new RegExp(`${key}:[\\s\\S]*?(?=\\n\\w|\\n$|$)`), `${key}:\n${deps.map(d => `  - ${d}`).join('\n')}\n`);
+    // --list: show all deps
+    if (isList) {
+        const npmDeps = readDeps('dependencies');
+        const pyDeps = readDeps('python_dependencies');
+        console.log(`Source: ${sourceName}\n`);
+        console.log(`npm (${npmDeps.length}):`);
+        for (const d of npmDeps)
+            console.log(`  ${d}`);
+        console.log(`\nPython (${pyDeps.length}):`);
+        for (const d of pyDeps)
+            console.log(`  ${d}`);
+        return;
     }
-    else {
-        yaml += `\n${key}:\n${deps.map(d => `  - ${d}`).join('\n')}\n`;
+    if (deps.length === 0) {
+        console.error('No packages specified.');
+        process.exit(1);
     }
-    fs_1.default.writeFileSync(yamlPath, yaml);
-    const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
-    if (isPython) {
-        console.log(`[pipe] Installing Python: ${deps.join(', ')}...`);
-        try {
-            run(`pip3 install ${deps.join(' ')}`, { cwd: repo, stdio: 'inherit' });
-            console.log(`[pipe] Python dependencies installed and added to YAML.`);
-        }
-        catch {
-            console.error(`[pipe] Failed to install Python dependencies.`);
-            process.exit(1);
+    // --remove: remove deps
+    if (isRemove) {
+        const current = readDeps(key);
+        const updated = current.filter(d => {
+            const name = d.split(/[@==]/)[0];
+            return !deps.includes(name) && !deps.includes(d);
+        });
+        writeDeps(key, updated);
+        console.log(`[pipe] Removed ${isPython ? 'Python' : 'npm'} deps: ${deps.join(', ')}`);
+        if (!isPython) {
+            const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
+            try {
+                run(`npm uninstall ${deps.join(' ')}`, { cwd: repo, stdio: 'inherit' });
+            }
+            catch { }
         }
+        return;
+    }
+    // Add deps
+    const current = readDeps(key);
+    for (const dep of deps) {
+        const name = dep.split(/[@==]/)[0];
+        // Replace if same package exists with different version
+        const idx = current.findIndex(d => d.split(/[@==]/)[0] === name);
+        if (idx >= 0)
+            current[idx] = dep;
+        else
+            current.push(dep);
+    }
+    writeDeps(key, current);
+    if (isPython) {
+        console.log(`[pipe] Python deps in YAML: ${current.join(', ')}`);
+        console.log(`[pipe] Will be installed on Linux Mint during pipe stage.`);
     }
     else {
         console.log(`[pipe] Installing npm: ${deps.join(', ')}...`);
+        const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
         try {
             run(`npm install ${deps.join(' ')}`, { cwd: repo, stdio: 'inherit' });
-            console.log(`[pipe] npm dependencies installed and added to YAML.`);
+            console.log(`[pipe] npm deps in YAML: ${current.join(', ')}`);
         }
         catch {
             console.error(`[pipe] Failed to install npm dependencies.`);
@@ -1807,9 +1873,10 @@ async function pipeTest(args) {
     if (pyDepsMatch) {
         const pyDeps = pyDepsMatch[1].split('\n').map(l => l.replace(/^\s*-\s*/, '').trim()).filter(Boolean);
         if (pyDeps.length > 0) {
-            console.log(`[pipe] Installing Python deps: ${pyDeps.join(', ')}...`);
+            console.log(`[pipe] Python deps: ${pyDeps.join(', ')} (installed on Linux Mint during stage)`);
+            // Try local install but don't fail if it can't (Mac blocks pip)
             try {
-                run(`pip3 install ${pyDeps.join(' ')}`, { cwd: repo, stdio: 'pipe' });
+                run(`pip3 install --user ${pyDeps.join(' ')} 2>/dev/null`, { cwd: repo, stdio: 'pipe' });
             }
             catch { }
         }
@@ -1955,7 +2022,7 @@ async function pipeStage(args) {
     // 3. Install deps on Linux Mint from YAML
     const yamlPath = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
     const yamlContent = fs_1.default.existsSync(yamlPath) ? fs_1.default.readFileSync(yamlPath, 'utf-8') : '';
-    const depsMatch = yamlContent.match(/dependencies:\n([\s\S]*?)(?=\n\w|\n$|$)/);
+    const depsMatch = yamlContent.match(/^dependencies:\n([\s\S]*?)(?=\n\w|\n$|$)/m);
     if (depsMatch) {
         const deps = depsMatch[1].split('\n').map(l => l.replace(/^\s*-\s*/, '').trim()).filter(Boolean);
         if (deps.length > 0) {
@@ -1974,14 +2041,15 @@ async function pipeStage(args) {
             }
         }
     }
-    // 3b. Install Python deps on Linux Mint
+    // 3b. Install Python deps on Linux Mint (using venv)
     const pyDepsMatch = yamlContent.match(/python_dependencies:\n([\s\S]*?)(?=\n\w|\n$|$)/);
     if (pyDepsMatch) {
         const pyDeps = pyDepsMatch[1].split('\n').map(l => l.replace(/^\s*-\s*/, '').trim()).filter(Boolean);
         if (pyDeps.length > 0) {
             console.log(`[pipe] Installing Python deps on Linux Mint: ${pyDeps.join(', ')}...`);
             try {
-                run(`ssh ${RUNNER} "pip3 install ${pyDeps.join(' ')}"`, { stdio: 'inherit' });
+                // Create venv if not exists, activate, install
+                run(`ssh ${RUNNER} "cd ${remoteRepo} && (test -d .venv || python3 -m venv .venv) && .venv/bin/pip install ${pyDeps.join(' ')}"`, { stdio: 'inherit' });
             }
             catch {
                 console.error('[pipe] Failed to install Python deps on Linux Mint');
@@ -1992,7 +2060,7 @@ async function pipeStage(args) {
     // 4. Run import on Linux Mint
     console.log(`[pipe] Running import on Linux Mint...`);
     try {
-        run(`ssh ${RUNNER} "cd ${remoteRepo} && set -a && source .env 2>/dev/null; source .dev.vars 2>/dev/null; set +a && R2_BUCKET_NAME=latinfo-data npx tsx src/imports/${sourceName}.ts"`, {
+        run(`ssh ${RUNNER} "cd ${remoteRepo} && set -a && source .env 2>/dev/null; source .dev.vars 2>/dev/null; test -f .venv/bin/activate && source .venv/bin/activate; set +a && R2_BUCKET_NAME=latinfo-data npx tsx src/imports/${sourceName}.ts"`, {
             stdio: 'inherit',
         });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "latinfo",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "description": "Tax registry & procurement API for Latin America. Query RUC, DNI, NIT, licitaciones from Peru & Colombia. Offline MPHF search, full OCDS data, updated daily.",
   "homepage": "https://latinfo.dev",
   "repository": {