latinfo 0.13.0 → 0.13.2

package/dist/index.js

@@ -47,7 +47,7 @@ const local_search_1 = require("./local-search");
 const client_search_1 = require("./client-search");
 const odis_search_1 = require("./odis-search");
 const mphf_search_1 = require("./mphf-search");
-const VERSION = '0.13.0';
+const VERSION = '0.13.2';
 const API_URL = process.env.LATINFO_API_URL || 'https://api.latinfo.dev';
 const GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID || 'Ov23li5fcQaiCsVtaMKK';
 const CONFIG_DIR = path_1.default.join(os_1.default.homedir(), '.latinfo');
@@ -1786,12 +1786,16 @@ async function pipeTest(args) {
         }
     }
     // Run import with --limit 100
-    const scriptPath = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
+    // Check YAML for custom import_script path
+    const importScriptMatch = yamlContent.match(/import_script:\s*(.+)/);
+    const customScript = importScriptMatch ? path_1.default.join(repo, importScriptMatch[1].trim()) : null;
+    const defaultScript = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
+    const scriptPath = customScript && fs_1.default.existsSync(customScript) ? customScript
+        : fs_1.default.existsSync(defaultScript) ? defaultScript : null;
     const easypipePath = path_1.default.join(repo, 'src', 'imports', 'easypipe.ts');
-    const useEasypipe = !fs_1.default.existsSync(scriptPath);
-    const cmd = useEasypipe
-        ? `npx tsx ${easypipePath} ${yamlPath} --limit 100 --local`
-        : `npx tsx ${scriptPath} --limit 100 --local`;
+    const cmd = scriptPath
+        ? `npx tsx ${scriptPath} --limit 100 --local`
+        : `npx tsx ${easypipePath} ${yamlPath} --limit 100 --local`;
     console.log(`[pipe] Gate 1: TEST (100 records)\n`);
     console.log(`Running: ${cmd}\n`);
     let output = '';
@@ -1852,12 +1856,17 @@ async function pipeValidate(args) {
     requireGate(status, 'test', 'validate');
     const repo = getRepoPath();
     const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
-    const scriptPath = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
+    const yamlPath = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
+    const yamlContent = fs_1.default.existsSync(yamlPath) ? fs_1.default.readFileSync(yamlPath, 'utf-8') : '';
+    const importScriptMatch = yamlContent.match(/import_script:\s*(.+)/);
+    const customScript = importScriptMatch ? path_1.default.join(repo, importScriptMatch[1].trim()) : null;
+    const defaultScript = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
+    const scriptPath = customScript && fs_1.default.existsSync(customScript) ? customScript
+        : fs_1.default.existsSync(defaultScript) ? defaultScript : null;
     const easypipePath = path_1.default.join(repo, 'src', 'imports', 'easypipe.ts');
-    const useEasypipe = !fs_1.default.existsSync(scriptPath);
-    const cmd = useEasypipe
-        ? `npx tsx ${easypipePath} ${path_1.default.join(repo, 'sources', `${sourceName}.yaml`)} --local`
-        : `npx tsx ${scriptPath} --local`;
+    const cmd = scriptPath
+        ? `npx tsx ${scriptPath} --local`
+        : `npx tsx ${easypipePath} ${yamlPath} --local`;
     console.log(`[pipe] Gate 2: VALIDATE (full import, local only — no R2 upload)\n`);
     console.log(`Running: ${cmd}\n`);
     try {
@@ -1914,7 +1923,29 @@ async function pipeStage(args) {
             run(`scp ${devVarsLocal} ${RUNNER}:${remoteRepo}/.dev.vars`, { stdio: 'pipe' });
     }
     catch { }
-    // 3. Run import on Linux Mint
+    // 3. Install deps on Linux Mint from YAML
+    const yamlPath = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
+    const yamlContent = fs_1.default.existsSync(yamlPath) ? fs_1.default.readFileSync(yamlPath, 'utf-8') : '';
+    const depsMatch = yamlContent.match(/dependencies:\n([\s\S]*?)(?=\n\w|\n$|$)/);
+    if (depsMatch) {
+        const deps = depsMatch[1].split('\n').map(l => l.replace(/^\s*-\s*/, '').trim()).filter(Boolean);
+        if (deps.length > 0) {
+            console.log(`[pipe] Installing deps on Linux Mint: ${deps.join(', ')}...`);
+            try {
+                run(`ssh ${RUNNER} "cd ${remoteRepo} && npm install ${deps.join(' ')}"`, { stdio: 'inherit', timeout: 120_000 });
+                // Install playwright browsers if needed
+                if (deps.some(d => d.includes('playwright'))) {
+                    console.log(`[pipe] Installing Playwright browser...`);
+                    run(`ssh ${RUNNER} "cd ${remoteRepo} && npx playwright install chromium --with-deps"`, { stdio: 'inherit', timeout: 120_000 });
+                }
+            }
+            catch {
+                console.error('[pipe] Failed to install deps on Linux Mint');
+                process.exit(1);
+            }
+        }
+    }
+    // 4. Run import on Linux Mint
     console.log(`[pipe] Running import on Linux Mint...`);
     try {
         run(`ssh ${RUNNER} "cd ${remoteRepo} && set -a && source .env 2>/dev/null; source .dev.vars 2>/dev/null; set +a && R2_BUCKET_NAME=latinfo-data npx tsx src/imports/${sourceName}.ts"`, {
@@ -2130,17 +2161,31 @@ async function pipePublish(args) {
     const { execSync: run } = await Promise.resolve().then(() => __importStar(require('child_process')));
     const RUNNER = 'f3mt0@100.109.82.87';
     console.log(`[pipe] Gate 4: PUBLISH\n`);
-    // 1. Git add + commit + push
+    // 1. Auto-generate sources.ts from YAMLs
+    console.log(`[pipe] Generating sources.ts...`);
+    try {
+        run(`npx tsx src/imports/generate-sources.ts`, { cwd: repo, stdio: 'inherit' });
+    }
+    catch {
+        console.error(`[pipe] Failed to generate sources.ts`);
+        process.exit(1);
+    }
+    // 2. Git add + commit + push
     console.log(`[pipe] Committing to repo...`);
-    const files = [`sources/${sourceName}.yaml`];
+    const files = [`sources/${sourceName}.yaml`, 'src/sources.ts'];
     const scriptPath = path_1.default.join(repo, 'src', 'imports', `${sourceName}.ts`);
-    if (fs_1.default.existsSync(scriptPath))
+    const yamlContent = fs_1.default.readFileSync(path_1.default.join(repo, 'sources', `${sourceName}.yaml`), 'utf-8');
+    const importScriptMatch = yamlContent.match(/import_script:\s*(.+)/);
+    const customScript = importScriptMatch ? importScriptMatch[1].trim() : null;
+    if (customScript && fs_1.default.existsSync(path_1.default.join(repo, customScript)))
+        files.push(customScript);
+    else if (fs_1.default.existsSync(scriptPath))
         files.push(`src/imports/${sourceName}.ts`);
+    const docsFile = `docs/sources/${sourceName}.md`;
+    if (fs_1.default.existsSync(path_1.default.join(repo, docsFile)))
+        files.push(docsFile);
     try {
-        const docsFile = `docs/sources/${sourceName}.md`;
-        if (fs_1.default.existsSync(path_1.default.join(repo, docsFile)))
-            files.push(docsFile);
-        run(`git add ${files.join(' ')} src/sources.ts .github/workflows/import.yml`, { cwd: repo, stdio: 'pipe' });
+        run(`git add ${files.join(' ')}`, { cwd: repo, stdio: 'pipe' });
         run(`git commit -m "Add data source: ${sourceName}"`, { cwd: repo, stdio: 'pipe' });
         run(`git push`, { cwd: repo, stdio: 'pipe' });
         console.log(`[pipe] Pushed to remote.`);
@@ -2170,7 +2215,7 @@ async function pipePublish(args) {
     // 4. Restart search server
     console.log(`[pipe] Restarting search server on Linux Mint...`);
     try {
-        run(`ssh ${RUNNER} "sudo systemctl restart search-server 2>/dev/null || echo 'No service yet'"`, { stdio: 'inherit' });
+        run(`ssh ${RUNNER} "sudo systemctl restart latinfo-search 2>/dev/null || echo 'No service yet'"`, { stdio: 'inherit' });
     }
     catch { }
     console.log(`\n[pipe] Gate 4 PASSED ✓`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "latinfo",
-  "version": "0.13.0",
+  "version": "0.13.2",
   "description": "Tax registry & procurement API for Latin America. Query RUC, DNI, NIT, licitaciones from Peru & Colombia. Offline MPHF search, full OCDS data, updated daily.",
   "homepage": "https://latinfo.dev",
   "repository": {