latinfo 0.20.5 → 0.20.7

package/dist/index.js

@@ -1587,6 +1587,7 @@ DATA SOURCES
     latinfo co rues registry --search <query>
 
 COMMANDS
+  sources                         List all data sources with stats
   login [--token <github_pat>]    GitHub OAuth or PAT login
   logout                          Remove credentials
   whoami                          Show authenticated user
@@ -4038,6 +4039,203 @@ function requireTeamAdmin() {
         process.exit(1);
     }
 }
+// --- Imports Status ---
+async function importsStatus() {
+    const { execSync: exec } = await Promise.resolve().then(() => __importStar(require('child_process')));
+    const ghRepo = 'carrerahaus/latinfo-api';
+    // Get recent workflow runs
+    const runsJson = exec(`gh run list --repo ${ghRepo} --workflow=import.yml --limit=5 --json databaseId,status,conclusion,createdAt`, { encoding: 'utf-8', stdio: 'pipe' }).trim();
+    const runs = JSON.parse(runsJson);
+    if (runs.length === 0) {
+        console.log('No recent import runs.');
+        return;
+    }
+    console.log('\n  IMPORT STATUS\n');
+    for (const run of runs) {
+        const jobsJson = exec(`gh run view ${run.databaseId} --repo ${ghRepo} --json jobs --jq '.jobs[] | "\\(.name)|\\(.status)|\\(.conclusion // "")"'`, { encoding: 'utf-8', stdio: 'pipe' }).trim();
+        const age = Math.floor((Date.now() - new Date(run.createdAt).getTime()) / 60000);
+        const ageStr = age < 60 ? `${age}m ago` : `${Math.floor(age / 60)}h ago`;
+        const runIcon = run.status === 'completed' ? (run.conclusion === 'success' ? '✅' : '❌') : '🔄';
+        console.log(`  ${runIcon} Run #${run.databaseId} (${ageStr})`);
+        for (const line of jobsJson.split('\n')) {
+            if (!line || line.includes('setup'))
+                continue;
+            const [name, status, conclusion] = line.split('|');
+            const icon = status === 'completed' ? (conclusion === 'success' ? '  ✅' : '  ❌') :
+                status === 'in_progress' ? '  🔄' :
+                    status === 'queued' ? '  ⏳' : '  ⬜';
+            const label = status === 'in_progress' ? 'running' :
+                status === 'queued' ? 'queued' :
+                    conclusion || status;
+            console.log(`  ${icon} ${name.padEnd(25)} ${label}`);
+        }
+        console.log();
+    }
+}
+// --- Sources ---
+async function sourcesCmd(args = []) {
+    const sourceName = args[0];
+    // Detail view: latinfo sources pe-servir-sanctions
+    if (sourceName && sourceName !== '--all') {
+        try {
+            const repo = getRepoPath();
+            const yamlPath = path_1.default.join(repo, 'sources', `${sourceName}.yaml`);
+            if (!fs_1.default.existsSync(yamlPath)) {
+                console.error(`Source not found: ${sourceName}`);
+                console.error(`Available sources:`);
+                const files = fs_1.default.readdirSync(path_1.default.join(repo, 'sources')).filter(f => f.endsWith('.yaml'));
+                for (const f of files)
+                    console.error(`  ${f.replace('.yaml', '')}`);
+                process.exit(1);
+            }
+            const content = fs_1.default.readFileSync(yamlPath, 'utf-8');
+            const get = (key) => content.match(new RegExp(`^${key}:\\s*(.+)`, 'm'))?.[1]?.trim() || '—';
+            const getQuoted = (key) => content.match(new RegExp(`${key}:\\s*"?([^"\\n]+)"?`))?.[1]?.trim() || '—';
+            // Get import metadata
+            let rows = '—', lastImport = '—';
+            try {
+                const adminSecret = requireAdmin();
+                const res = await fetch(`${API_URL}/admin/imports`, { headers: { Authorization: `Bearer ${adminSecret}` } });
+                if (res.ok) {
+                    const imports = await res.json();
+                    // Match by source name (handle co-rues vs co-rues-registry)
+                    const meta = imports.find(i => sourceName.startsWith(i.source) || i.source.startsWith(sourceName));
+                    if (meta) {
+                        rows = meta.rows?.toLocaleString() || '—';
+                        lastImport = meta.importedAt ? `${meta.importedAt.slice(0, 16)} (${meta.ageHours}h ago)` : '—';
+                    }
+                }
+            }
+            catch { }
+            // Parse fields
+            const fields = [];
+            const fieldMatches = content.matchAll(/- name: (\S+)/g);
+            for (const m of fieldMatches)
+                fields.push(m[1]);
+            console.log(`
+  ${sourceName}
+
+  Country:      ${get('country') === 'pe' ? 'Peru' : get('country') === 'co' ? 'Colombia' : get('country')}
+  Institution:  ${get('institution').toUpperCase()}
+  Dataset:      ${get('dataset')}
+  URL:          ${get('url')}
+  Format:       ${get('format')}
+  Schedule:     ${get('schedule')}
+  Records:      ${rows}
+  Last import:  ${lastImport}
+  ID:           ${getQuoted('name')} (${getQuoted('length')} digits)
+  Fields:       ${fields.join(', ')}
+  Script:       ${get('import_script') !== '—' ? get('import_script') : `src/imports/${sourceName}.ts`}
+  Min rows:     ${get('min_rows')}
+`);
+        }
+        catch (e) {
+            console.error(`Cannot read source: ${e.message}`);
+        }
+        return;
+    }
+    // List view: latinfo sources (or --all)
+    // Get import metadata dynamically from admin endpoint
+    let data = [];
+    try {
+        const adminSecret = requireAdmin();
+        const res = await fetch(`${API_URL}/admin/imports`, { headers: { Authorization: `Bearer ${adminSecret}` } });
+        if (res.ok)
+            data = await res.json();
+    }
+    catch {
+        // Not admin — try reading source YAMLs from repo
+    }
+    if (data.length === 0) {
+        // Fallback: read sources/*.yaml from repo
+        try {
+            const repo = getRepoPath();
+            const sourcesDir = path_1.default.join(repo, 'sources');
+            const files = fs_1.default.readdirSync(sourcesDir).filter(f => f.endsWith('.yaml'));
+            for (const f of files) {
+                const content = fs_1.default.readFileSync(path_1.default.join(sourcesDir, f), 'utf-8');
+                const name = f.replace('.yaml', '');
+                const country = content.match(/country:\s*(\S+)/)?.[1] || '?';
+                const institution = content.match(/institution:\s*(\S+)/)?.[1] || '?';
+                const dataset = content.match(/dataset:\s*(\S+)/)?.[1] || '?';
+                const idName = content.match(/name:\s*(\S+)/)?.[1] || 'id';
+                data.push({ source: name, country, institution, dataset, idName, rows: null, ageHours: null });
+            }
+        }
+        catch { }
+    }
+    if (data.length === 0) {
+        console.log('No sources found.');
+        return;
+    }
+    // Read schedule + change detection from YAMLs
+    const yamlMeta = {};
+    try {
+        const repo = getRepoPath();
+        const sourcesDir = path_1.default.join(repo, 'sources');
+        for (const f of fs_1.default.readdirSync(sourcesDir).filter(f => f.endsWith('.yaml'))) {
+            const content = fs_1.default.readFileSync(path_1.default.join(sourcesDir, f), 'utf-8');
+            const name = f.replace('.yaml', '');
+            const schedule = content.match(/^schedule:\s*(\S+)/m)?.[1] || '?';
+            const url = content.match(/^url:\s*(\S+)/m)?.[1] || '';
+            // Sources with HTTP URLs (not APIs/scraping) can do HEAD request change detection
+            const hasCheck = url.startsWith('http') && !content.includes('format: api');
+            yamlMeta[name] = { schedule, hasCheck };
+        }
+    }
+    catch { }
+    // Colors
+    const R = '\x1b[31m';
+    const G = '\x1b[32m';
+    const Y = '\x1b[33m';
+    const D = '\x1b[90m';
+    const X = '\x1b[0m';
+    const noColor = !process.stdout.isTTY;
+    console.log('\n  SOURCES\n');
+    for (const s of data) {
+        const parts = s.source.split('-');
+        const country = s.country || (parts[0] === 'pe' ? 'Peru' : parts[0] === 'co' ? 'Colombia' : parts[0]);
+        const institution = s.institution || parts.slice(1, -1).join('-');
+        const dataset = s.dataset || parts[parts.length - 1];
+        const label = `${country} — ${institution.toUpperCase()} ${dataset}`;
+        const rows = s.rows ? s.rows.toLocaleString() : '—';
+        // Age with minutes support + colors
+        const ageH = s.ageHours ?? -1;
+        let ageStr;
+        let ageColor;
+        if (ageH < 0) {
+            ageStr = '—';
+            ageColor = D;
+        }
+        else if (ageH < 1) {
+            const mins = Math.round(ageH * 60);
+            ageStr = `${mins}m ago`;
+            ageColor = G;
+        }
+        else if (ageH < 24) {
+            ageStr = `${ageH}h ago`;
+            ageColor = G;
+        }
+        else if (ageH < 72) {
+            ageStr = `${Math.floor(ageH / 24)}d ago`;
+            ageColor = Y;
+        }
+        else {
+            ageStr = `${Math.floor(ageH / 24)}d ago`;
+            ageColor = R;
+        }
+        const meta = yamlMeta[s.source] || yamlMeta[s.source.replace(/-registry$/, '')] || {};
+        const schedule = (meta.schedule || '?').padEnd(7);
+        const check = meta.hasCheck ? 'yes' : '—';
+        if (noColor) {
+            console.log(`  ${label.padEnd(32)} ${rows.padStart(12)}  ${ageStr.padEnd(8)}  ${schedule}  ${check.padEnd(5)}`);
+        }
+        else {
+            console.log(`  ${label.padEnd(32)} ${rows.padStart(12)}  ${ageColor}${ageStr.padEnd(8)}${X}  ${D}${schedule}${X}  ${meta.hasCheck ? G : D}${check.padEnd(5)}${X}`);
+        }
+    }
+    console.log();
+}
 // --- Team & Tasks ---
 async function teamCmd(args) {
     const sub = args[0];
@@ -4407,6 +4605,9 @@ else {
         case 'users':
             users().catch(e => { console.error(e); process.exit(1); });
             break;
+        case 'sources':
+            sourcesCmd(args).catch(e => { console.error(e); process.exit(1); });
+            break;
         case 'plan':
             plan().catch(e => { console.error(e); process.exit(1); });
             break;
@@ -4415,13 +4616,19 @@ else {
             break;
         // Team-only commands
         case 'imports':
-            requireTeam();
-            if (args[0] === 'run')
-                importsRun(args[1] || 'all').catch(e => { console.error(e); process.exit(1); });
-            else if (args[0] === 'report')
-                importsReport(parseInt(args[1]) || 7).catch(e => { console.error(e); process.exit(1); });
-            else
-                imports().catch(e => { console.error(e); process.exit(1); });
+            if (args[0] === 'status') {
+                requireTeam();
+                importsStatus().catch(e => { console.error(e); process.exit(1); });
+            }
+            else {
+                requireTeamAdmin();
+                if (args[0] === 'run')
+                    importsRun(args[1] || 'all').catch(e => { console.error(e); process.exit(1); });
+                else if (args[0] === 'report')
+                    importsReport(parseInt(args[1]) || 7).catch(e => { console.error(e); process.exit(1); });
+                else
+                    imports().catch(e => { console.error(e); process.exit(1); });
+            }
             break;
         case 'costs':
             requireTeamAdmin();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "latinfo",
-  "version": "0.20.5",
+  "version": "0.20.7",
   "description": "Tax registry & procurement API for Latin America. Query RUC, DNI, NIT, licitaciones from Peru & Colombia. Offline MPHF search, full OCDS data, updated daily.",
   "homepage": "https://latinfo.dev",
   "repository": {