latinfo 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +279 -76
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -47,7 +47,7 @@ const local_search_1 = require("./local-search");
|
|
|
47
47
|
const client_search_1 = require("./client-search");
|
|
48
48
|
const odis_search_1 = require("./odis-search");
|
|
49
49
|
const mphf_search_1 = require("./mphf-search");
|
|
50
|
-
const VERSION = '0.
|
|
50
|
+
const VERSION = '0.17.0';
|
|
51
51
|
const API_URL = process.env.LATINFO_API_URL || 'https://api.latinfo.dev';
|
|
52
52
|
const GITHUB_CLIENT_ID = process.env.GITHUB_CLIENT_ID || 'Ov23li5fcQaiCsVtaMKK';
|
|
53
53
|
const CONFIG_DIR = path_1.default.join(os_1.default.homedir(), '.latinfo');
|
|
@@ -748,39 +748,122 @@ function costsSimulate(usersStr, rpmStr, proPctStr) {
|
|
|
748
748
|
printCosts({ users, pro_users: proUsers, requests, cf_tier: cfTier, cf_cost: cfCost, revenue, margin, safe: margin >= 0 });
|
|
749
749
|
}
|
|
750
750
|
// --- Bench ---
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
'
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
751
|
+
// Seed queries used to discover real data from the API
|
|
752
|
+
const SEED_QUERIES = ['banco', 'empresa', 'servicios', 'comercial', 'grupo'];
|
|
753
|
+
function discoverBenchSources(filterSource) {
|
|
754
|
+
const repo = getRepoPath();
|
|
755
|
+
const sourcesDir = path_1.default.join(repo, 'sources');
|
|
756
|
+
if (!fs_1.default.existsSync(sourcesDir))
|
|
757
|
+
return [];
|
|
758
|
+
const yamls = fs_1.default.readdirSync(sourcesDir).filter(f => f.endsWith('.yaml'));
|
|
759
|
+
const sources = [];
|
|
760
|
+
for (const file of yamls) {
|
|
761
|
+
const content = fs_1.default.readFileSync(path_1.default.join(sourcesDir, file), 'utf-8');
|
|
762
|
+
const nameMatch = content.match(/^name:\s*(.+)/m);
|
|
763
|
+
const countryMatch = content.match(/^country:\s*(.+)/m);
|
|
764
|
+
const institutionMatch = content.match(/^institution:\s*(.+)/m);
|
|
765
|
+
const datasetMatch = content.match(/^dataset:\s*(.+)/m);
|
|
766
|
+
const idNameMatch = content.match(/primary_id:\s*\n\s*name:\s*(.+)/);
|
|
767
|
+
const idLenMatch = content.match(/primary_id:[\s\S]*?length:\s*(\d+)/);
|
|
768
|
+
const smokeIdMatch = content.match(/smoke_test:\s*\n\s*id:\s*"?([^"\n]+)"?/);
|
|
769
|
+
if (!nameMatch || !countryMatch || !institutionMatch || !datasetMatch)
|
|
770
|
+
continue;
|
|
771
|
+
const name = nameMatch[1].trim();
|
|
772
|
+
if (filterSource && name !== filterSource)
|
|
773
|
+
continue;
|
|
774
|
+
const routePath = `/${countryMatch[1].trim()}/${institutionMatch[1].trim()}/${datasetMatch[1].trim()}`;
|
|
775
|
+
const primaryId = {
|
|
776
|
+
name: idNameMatch ? idNameMatch[1].trim() : 'id',
|
|
777
|
+
length: idLenMatch ? parseInt(idLenMatch[1]) : 10,
|
|
778
|
+
};
|
|
779
|
+
const smokeId = smokeIdMatch ? smokeIdMatch[1].trim() : undefined;
|
|
780
|
+
const lookupIds = [];
|
|
781
|
+
if (smokeId)
|
|
782
|
+
lookupIds.push(smokeId);
|
|
783
|
+
sources.push({ name, routePath, primaryId, smokeId, lookupIds, searchQueries: [] });
|
|
784
|
+
}
|
|
785
|
+
return sources;
|
|
786
|
+
}
|
|
787
|
+
/** Generate a non-existent ID based on the primary ID pattern */
|
|
788
|
+
function generateFakeId(length) {
|
|
789
|
+
// All 9s — unlikely to be a real ID
|
|
790
|
+
return '9'.repeat(length);
|
|
791
|
+
}
|
|
792
|
+
/** Seed lookup IDs + search queries from real API data */
|
|
793
|
+
async function seedBenchQueries(source, apiKey) {
|
|
794
|
+
const headers = { Authorization: `Bearer ${apiKey}` };
|
|
795
|
+
const names = [];
|
|
796
|
+
// Fetch real records via search
|
|
797
|
+
for (const seed of SEED_QUERIES) {
|
|
798
|
+
try {
|
|
799
|
+
const res = await fetch(`${API_URL}${source.routePath}/search?q=${encodeURIComponent(seed)}&limit=20`, {
|
|
800
|
+
headers, signal: AbortSignal.timeout(10_000),
|
|
801
|
+
});
|
|
802
|
+
if (!res.ok)
|
|
803
|
+
continue;
|
|
804
|
+
const data = await res.json();
|
|
805
|
+
for (const r of data) {
|
|
806
|
+
const id = r.id || r[source.primaryId.name];
|
|
807
|
+
if (id && !source.lookupIds.includes(String(id)))
|
|
808
|
+
source.lookupIds.push(String(id));
|
|
809
|
+
// Extract the search field (first string field that isn't the ID)
|
|
810
|
+
const name = Object.values(r).find((v, i) => typeof v === 'string' && i > 0);
|
|
811
|
+
if (name && name.length > 3)
|
|
812
|
+
names.push(name);
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
catch { }
|
|
816
|
+
if (source.lookupIds.length >= 20 && names.length >= 20)
|
|
817
|
+
break;
|
|
818
|
+
}
|
|
819
|
+
// Add non-existent IDs for edge case testing
|
|
820
|
+
for (let i = 0; i < 5; i++) {
|
|
821
|
+
source.lookupIds.push(generateFakeId(source.primaryId.length));
|
|
822
|
+
}
|
|
823
|
+
// Generate diverse search queries from real names
|
|
824
|
+
const uniqueNames = [...new Set(names)];
|
|
825
|
+
for (const name of uniqueNames.slice(0, 10)) {
|
|
826
|
+
// Full name (easy)
|
|
827
|
+
source.searchQueries.push(name);
|
|
828
|
+
}
|
|
829
|
+
for (const name of uniqueNames.slice(0, 10)) {
|
|
830
|
+
// Single token (medium)
|
|
831
|
+
const tokens = name.split(/\s+/).filter(t => t.length >= 3);
|
|
832
|
+
if (tokens.length > 0)
|
|
833
|
+
source.searchQueries.push(tokens[0]);
|
|
834
|
+
}
|
|
835
|
+
for (const name of uniqueNames.slice(0, 10)) {
|
|
836
|
+
// 3-letter prefix (hard)
|
|
837
|
+
const token = name.split(/\s+/).find(t => t.length >= 4);
|
|
838
|
+
if (token)
|
|
839
|
+
source.searchQueries.push(token.slice(0, 3));
|
|
840
|
+
}
|
|
841
|
+
for (const name of uniqueNames.slice(0, 5)) {
|
|
842
|
+
// Multi-token substring (hard)
|
|
843
|
+
const tokens = name.split(/\s+/).filter(t => t.length >= 3);
|
|
844
|
+
if (tokens.length >= 2)
|
|
845
|
+
source.searchQueries.push(tokens.slice(0, 2).join(' '));
|
|
846
|
+
}
|
|
847
|
+
// Non-existent search queries (edge case)
|
|
848
|
+
source.searchQueries.push('xyznonexistent', 'qqq999', 'zzznodata');
|
|
849
|
+
// Fallback: if API returned nothing, use seed queries
|
|
850
|
+
if (source.searchQueries.length < 5) {
|
|
851
|
+
source.searchQueries.push(...SEED_QUERIES);
|
|
852
|
+
}
|
|
853
|
+
}
|
|
780
854
|
async function benchStress(args) {
|
|
781
855
|
const config = requireAuth();
|
|
782
856
|
const durationSec = parseInt(parseFlag(args, '--duration') || '120');
|
|
783
857
|
const maxVUs = parseInt(parseFlag(args, '--max-vus') || '500');
|
|
858
|
+
const filterSource = parseFlag(args, '--source');
|
|
859
|
+
const benchSources = discoverBenchSources(filterSource || undefined);
|
|
860
|
+
if (benchSources.length === 0) {
|
|
861
|
+
console.error('No sources found. Check sources/ directory.');
|
|
862
|
+
process.exit(1);
|
|
863
|
+
}
|
|
864
|
+
// Seed lookup IDs from API
|
|
865
|
+
console.log(` Seeding queries from ${benchSources.length} sources...`);
|
|
866
|
+
await Promise.all(benchSources.map(s => seedBenchQueries(s, config.api_key)));
|
|
784
867
|
const stages = [
|
|
785
868
|
{ name: 'warmup', vus: 10, duration: Math.floor(durationSec * 0.08) },
|
|
786
869
|
{ name: 'ramp', vus: Math.floor(maxVUs * 0.5), duration: Math.floor(durationSec * 0.17) },
|
|
@@ -789,16 +872,21 @@ async function benchStress(args) {
|
|
|
789
872
|
{ name: 'hold', vus: maxVUs, duration: Math.floor(durationSec * 0.25) },
|
|
790
873
|
{ name: 'cool', vus: 10, duration: Math.floor(durationSec * 0.08) },
|
|
791
874
|
];
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
875
|
+
// Build endpoints dynamically from discovered sources
|
|
876
|
+
const endpoints = [];
|
|
877
|
+
for (const s of benchSources) {
|
|
878
|
+
if (s.lookupIds.length > 0) {
|
|
879
|
+
for (const id of s.lookupIds.slice(0, 10)) {
|
|
880
|
+
endpoints.push({ url: `${API_URL}${s.routePath}/${s.primaryId.name}/${id}`, type: `${s.name}/lookup` });
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
for (const q of s.searchQueries.slice(0, 10)) {
|
|
884
|
+
endpoints.push({ url: `${API_URL}${s.routePath}/search?q=${encodeURIComponent(q)}&limit=5`, type: `${s.name}/search` });
|
|
885
|
+
}
|
|
886
|
+
}
|
|
799
887
|
const headers = { Authorization: `Bearer ${config.api_key}` };
|
|
800
888
|
const results = [];
|
|
801
|
-
console.log(`\n STRESS TEST — ${durationSec}s, max ${maxVUs} VUs,
|
|
889
|
+
console.log(`\n STRESS TEST — ${durationSec}s, max ${maxVUs} VUs, ${benchSources.map(s => s.name).join(' + ')}\n`);
|
|
802
890
|
console.log(` Stage VUs Duration`);
|
|
803
891
|
for (const s of stages)
|
|
804
892
|
console.log(` ${s.name.padEnd(10)} ${String(s.vus).padEnd(5)} ${s.duration}s`);
|
|
@@ -886,6 +974,7 @@ async function benchStress(args) {
|
|
|
886
974
|
console.log(`\n${'═'.repeat(60)}`);
|
|
887
975
|
console.log(` STRESS TEST RESULTS`);
|
|
888
976
|
console.log(`${'═'.repeat(60)}\n`);
|
|
977
|
+
console.log(` Sources: ${benchSources.map(s => s.name).join(', ')}`);
|
|
889
978
|
console.log(` Duration: ${dur}s`);
|
|
890
979
|
console.log(` Max VUs: ${maxVUs}`);
|
|
891
980
|
console.log(` Total reqs: ${total.toLocaleString()}`);
|
|
@@ -918,10 +1007,10 @@ async function benchStress(args) {
|
|
|
918
1007
|
}
|
|
919
1008
|
console.log();
|
|
920
1009
|
console.log(` Per Endpoint`);
|
|
921
|
-
console.log(` ${'Type'.padEnd(
|
|
1010
|
+
console.log(` ${'Type'.padEnd(25)} ${'Reqs'.padEnd(8)} ${'p50'.padEnd(8)} ${'p95'.padEnd(8)} p99`);
|
|
922
1011
|
for (const [t, d] of Object.entries(allByType)) {
|
|
923
1012
|
const sl = d.latencies.sort((a, b) => a - b);
|
|
924
|
-
console.log(` ${t.padEnd(
|
|
1013
|
+
console.log(` ${t.padEnd(25)} ${String(d.count).padEnd(8)} ${(pct(sl, 0.5) + 'ms').padEnd(8)} ${(pct(sl, 0.95) + 'ms').padEnd(8)} ${pct(sl, 0.99)}ms`);
|
|
925
1014
|
}
|
|
926
1015
|
console.log();
|
|
927
1016
|
console.log(` HTTP Status: ${Object.entries(allStat).map(([s, c]) => `${s}: ${c}`).join(' ')}`);
|
|
@@ -996,44 +1085,48 @@ async function bench(args) {
|
|
|
996
1085
|
const count = countIdx !== -1 ? parseInt(args[countIdx + 1]) || 100 : 100;
|
|
997
1086
|
const concurrencyIdx = args.indexOf('--concurrency');
|
|
998
1087
|
const concurrency = concurrencyIdx !== -1 ? parseInt(args[concurrencyIdx + 1]) || 10 : 10;
|
|
999
|
-
const
|
|
1000
|
-
const
|
|
1001
|
-
const
|
|
1002
|
-
const type = typeIdx !== -1 ? args[typeIdx + 1] : (country === 'pe' ? 'ruc' : 'nit');
|
|
1088
|
+
const filterSource = parseFlag(args, '--source');
|
|
1089
|
+
const searchOnly = args.includes('--search-only');
|
|
1090
|
+
const lookupOnly = args.includes('--lookup-only');
|
|
1003
1091
|
const config = requireAuth();
|
|
1004
|
-
const
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
console.error(`Unknown combination: --country ${country} --type ${type}`);
|
|
1008
|
-
console.error(`Supported: ${Object.keys(BENCH_SAMPLES).map(k => '--country ' + k.replace('/', ' --type ')).join(', ')}`);
|
|
1092
|
+
const benchSources = discoverBenchSources(filterSource || undefined);
|
|
1093
|
+
if (benchSources.length === 0) {
|
|
1094
|
+
console.error('No sources found. Check sources/ directory or use --source <name>.');
|
|
1009
1095
|
process.exit(1);
|
|
1010
1096
|
}
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1097
|
+
// Seed queries + IDs from real API data
|
|
1098
|
+
if (!jsonFlag)
|
|
1099
|
+
console.log(` Seeding queries from ${benchSources.length} sources...`);
|
|
1100
|
+
await Promise.all(benchSources.map(s => seedBenchQueries(s, config.api_key)));
|
|
1101
|
+
// Build URL list
|
|
1102
|
+
const urls = [];
|
|
1103
|
+
for (const s of benchSources) {
|
|
1104
|
+
if (!searchOnly && s.lookupIds.length > 0) {
|
|
1105
|
+
for (const id of s.lookupIds) {
|
|
1106
|
+
urls.push({ url: `${API_URL}${s.routePath}/${s.primaryId.name}/${id}`, type: `${s.name}/lookup` });
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
if (!lookupOnly) {
|
|
1110
|
+
for (const q of s.searchQueries) {
|
|
1111
|
+
urls.push({ url: `${API_URL}${s.routePath}/search?q=${encodeURIComponent(q)}&limit=5`, type: `${s.name}/search` });
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
const sourceNames = benchSources.map(s => s.name).join(', ');
|
|
1025
1116
|
if (!jsonFlag)
|
|
1026
|
-
console.log(`\
|
|
1117
|
+
console.log(`\n Bench: ${sourceNames} — ${count} requests, ${concurrency} concurrent\n`);
|
|
1027
1118
|
const latencies = [];
|
|
1028
1119
|
const statusCounts = {};
|
|
1120
|
+
const byType = {};
|
|
1029
1121
|
let completed = 0;
|
|
1030
1122
|
const startAll = Date.now();
|
|
1123
|
+
const tasks = Array.from({ length: count }, (_, i) => urls[i % urls.length]);
|
|
1031
1124
|
for (let i = 0; i < tasks.length; i += concurrency) {
|
|
1032
1125
|
const batch = tasks.slice(i, i + concurrency);
|
|
1033
|
-
await Promise.all(batch.map(async (
|
|
1126
|
+
await Promise.all(batch.map(async (task) => {
|
|
1034
1127
|
const t0 = Date.now();
|
|
1035
1128
|
try {
|
|
1036
|
-
const res = await fetch(
|
|
1129
|
+
const res = await fetch(task.url, {
|
|
1037
1130
|
headers: { Authorization: `Bearer ${config.api_key}` },
|
|
1038
1131
|
signal: AbortSignal.timeout(15_000),
|
|
1039
1132
|
});
|
|
@@ -1044,6 +1137,10 @@ async function bench(args) {
|
|
|
1044
1137
|
latencies.push(Date.now() - t0);
|
|
1045
1138
|
statusCounts[0] = (statusCounts[0] || 0) + 1;
|
|
1046
1139
|
}
|
|
1140
|
+
if (!byType[task.type])
|
|
1141
|
+
byType[task.type] = { count: 0, latencies: [] };
|
|
1142
|
+
byType[task.type].count++;
|
|
1143
|
+
byType[task.type].latencies.push(latencies[latencies.length - 1]);
|
|
1047
1144
|
completed++;
|
|
1048
1145
|
if (process.stdout.isTTY && !jsonFlag) {
|
|
1049
1146
|
const pct = Math.floor(completed / count * 30);
|
|
@@ -1060,11 +1157,14 @@ async function bench(args) {
|
|
|
1060
1157
|
const rps = (count / (totalMs / 1000)).toFixed(1);
|
|
1061
1158
|
if (jsonFlag) {
|
|
1062
1159
|
console.log(JSON.stringify({
|
|
1063
|
-
|
|
1064
|
-
total_ms: totalMs,
|
|
1065
|
-
rps: parseFloat(rps),
|
|
1066
|
-
status: statusCounts,
|
|
1160
|
+
sources: benchSources.map(s => s.name), count, concurrency,
|
|
1161
|
+
total_ms: totalMs, rps: parseFloat(rps), status: statusCounts,
|
|
1067
1162
|
latency: { p50: p(0.5), p95: p(0.95), p99: p(0.99), max: latencies[latencies.length - 1] },
|
|
1163
|
+
byType: Object.fromEntries(Object.entries(byType).map(([t, d]) => {
|
|
1164
|
+
const sl = d.latencies.sort((a, b) => a - b);
|
|
1165
|
+
const pp = (pct) => sl[Math.floor(sl.length * pct)] ?? 0;
|
|
1166
|
+
return [t, { count: d.count, p50: pp(0.5), p95: pp(0.95), p99: pp(0.99) }];
|
|
1167
|
+
})),
|
|
1068
1168
|
}));
|
|
1069
1169
|
return;
|
|
1070
1170
|
}
|
|
@@ -1080,6 +1180,7 @@ async function bench(args) {
|
|
|
1080
1180
|
rateLimit ? `429: ${rateLimit}` : '',
|
|
1081
1181
|
errors ? `Errors: ${errors}` : '',
|
|
1082
1182
|
].filter(Boolean).join(' ');
|
|
1183
|
+
console.log(` Sources: ${sourceNames}`);
|
|
1083
1184
|
console.log(` Total: ${(totalMs / 1000).toFixed(1)}s Req/sec: ${rps}`);
|
|
1084
1185
|
console.log(` Status: ${statusLine}`);
|
|
1085
1186
|
console.log();
|
|
@@ -1087,6 +1188,65 @@ async function bench(args) {
|
|
|
1087
1188
|
console.log(` p95: ${p(0.95)}ms`);
|
|
1088
1189
|
console.log(` p99: ${p(0.99)}ms`);
|
|
1089
1190
|
console.log(` max: ${latencies[latencies.length - 1]}ms`);
|
|
1191
|
+
if (Object.keys(byType).length > 1) {
|
|
1192
|
+
console.log();
|
|
1193
|
+
console.log(` Per Endpoint`);
|
|
1194
|
+
console.log(` ${'Type'.padEnd(25)} ${'Reqs'.padEnd(8)} ${'p50'.padEnd(8)} ${'p95'.padEnd(8)} p99`);
|
|
1195
|
+
for (const [t, d] of Object.entries(byType)) {
|
|
1196
|
+
const sl = d.latencies.sort((a, b) => a - b);
|
|
1197
|
+
const pp = (pct) => sl[Math.floor(sl.length * pct)] ?? 0;
|
|
1198
|
+
console.log(` ${t.padEnd(25)} ${String(d.count).padEnd(8)} ${(pp(0.5) + 'ms').padEnd(8)} ${(pp(0.95) + 'ms').padEnd(8)} ${pp(0.99)}ms`);
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
1202
|
+
/** Run production bench: 500 concurrent against api.latinfo.dev, returns success rate */
|
|
1203
|
+
async function benchProduction(sourceName, apiKey, concurrency = 500) {
|
|
1204
|
+
const benchSources = discoverBenchSources(sourceName);
|
|
1205
|
+
if (benchSources.length === 0)
|
|
1206
|
+
throw new Error(`Source ${sourceName} not found`);
|
|
1207
|
+
const source = benchSources[0];
|
|
1208
|
+
// Seed IDs
|
|
1209
|
+
await seedBenchQueries(source, apiKey);
|
|
1210
|
+
// Build mixed URL list (search + lookup)
|
|
1211
|
+
const urls = [];
|
|
1212
|
+
for (const q of source.searchQueries) {
|
|
1213
|
+
urls.push(`${API_URL}${source.routePath}/search?q=${encodeURIComponent(q)}&limit=5`);
|
|
1214
|
+
}
|
|
1215
|
+
for (const id of source.lookupIds) {
|
|
1216
|
+
urls.push(`${API_URL}${source.routePath}/${source.primaryId.name}/${id}`);
|
|
1217
|
+
}
|
|
1218
|
+
const total = concurrency;
|
|
1219
|
+
const headers = { Authorization: `Bearer ${apiKey}` };
|
|
1220
|
+
let idx = 0, success = 0, fails = 0;
|
|
1221
|
+
const lats = [];
|
|
1222
|
+
const t0 = Date.now();
|
|
1223
|
+
async function worker() {
|
|
1224
|
+
while (idx < total) {
|
|
1225
|
+
const i = idx++;
|
|
1226
|
+
const url = urls[i % urls.length];
|
|
1227
|
+
const start = Date.now();
|
|
1228
|
+
try {
|
|
1229
|
+
const res = await fetch(url, { headers, signal: AbortSignal.timeout(15_000) });
|
|
1230
|
+
lats.push(Date.now() - start);
|
|
1231
|
+
if (res.status === 200 || res.status === 404)
|
|
1232
|
+
success++;
|
|
1233
|
+
else
|
|
1234
|
+
fails++;
|
|
1235
|
+
}
|
|
1236
|
+
catch {
|
|
1237
|
+
lats.push(Date.now() - start);
|
|
1238
|
+
fails++;
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
await Promise.all(Array.from({ length: concurrency }, () => worker()));
|
|
1243
|
+
lats.sort((a, b) => a - b);
|
|
1244
|
+
const p = (pct) => lats[Math.floor(lats.length * pct)] ?? 0;
|
|
1245
|
+
return {
|
|
1246
|
+
success_rate: (success / total) * 100,
|
|
1247
|
+
p50: p(0.5), p95: p(0.95), p99: p(0.99),
|
|
1248
|
+
qps: Math.round(total / ((Date.now() - t0) / 1000)),
|
|
1249
|
+
};
|
|
1090
1250
|
}
|
|
1091
1251
|
// --- Licitaciones ---
|
|
1092
1252
|
function parseFlag(args, flag) {
|
|
@@ -2356,18 +2516,61 @@ async function pipePublish(args) {
|
|
|
2356
2516
|
catch {
|
|
2357
2517
|
console.log(`[pipe] Could not trigger workflow automatically.`);
|
|
2358
2518
|
}
|
|
2359
|
-
// 4.
|
|
2360
|
-
console.log(`[pipe]
|
|
2519
|
+
// 4. Update search server SOURCES env + restart
|
|
2520
|
+
console.log(`[pipe] Updating search server SOURCES on Linux Mint...`);
|
|
2521
|
+
try {
|
|
2522
|
+
// Read current SOURCES from systemd service
|
|
2523
|
+
const currentSources = run(`ssh -o ConnectTimeout=10 ${RUNNER} "grep -oP 'SOURCES=\\K.*' /etc/systemd/system/latinfo-search.service 2>/dev/null || echo ''"`, {
|
|
2524
|
+
encoding: 'utf-8', stdio: 'pipe',
|
|
2525
|
+
}).trim().replace(/^"|"$/g, '');
|
|
2526
|
+
const sourceList = currentSources ? currentSources.split(',').map(s => s.trim()) : [];
|
|
2527
|
+
if (!sourceList.includes(sourceName)) {
|
|
2528
|
+
sourceList.push(sourceName);
|
|
2529
|
+
const newSources = sourceList.join(',');
|
|
2530
|
+
console.log(`[pipe] Adding ${sourceName} to SOURCES: ${newSources}`);
|
|
2531
|
+
run(`ssh ${RUNNER} "sudo sed -i 's|^Environment=.*SOURCES=.*|Environment=SOURCES=${newSources}|' /etc/systemd/system/latinfo-search.service && sudo systemctl daemon-reload"`, { stdio: 'pipe' });
|
|
2532
|
+
}
|
|
2533
|
+
else {
|
|
2534
|
+
console.log(`[pipe] ${sourceName} already in SOURCES.`);
|
|
2535
|
+
}
|
|
2536
|
+
run(`ssh ${RUNNER} "sudo systemctl restart latinfo-search"`, { stdio: 'inherit' });
|
|
2537
|
+
console.log(`[pipe] Search server restarted.`);
|
|
2538
|
+
}
|
|
2539
|
+
catch {
|
|
2540
|
+
console.log(`[pipe] Could not update search server (not critical).`);
|
|
2541
|
+
}
|
|
2542
|
+
// 5. Production bench: 500 concurrent against api.latinfo.dev
|
|
2543
|
+
console.log(`\n[pipe] Running production bench (500 concurrent)...`);
|
|
2361
2544
|
try {
|
|
2362
|
-
|
|
2545
|
+
const config = loadConfig();
|
|
2546
|
+
if (!config?.api_key)
|
|
2547
|
+
throw new Error('No API key');
|
|
2548
|
+
const bench = await benchProduction(sourceName, config.api_key, 500);
|
|
2549
|
+
console.log(`\n Production bench: ${bench.qps} q/s, ${bench.success_rate.toFixed(1)}% success`);
|
|
2550
|
+
console.log(` p50: ${bench.p50}ms p95: ${bench.p95}ms p99: ${bench.p99}ms`);
|
|
2551
|
+
if (bench.success_rate < 99.9) {
|
|
2552
|
+
console.error(`\n[pipe] PRODUCTION BENCH FAILED — ${bench.success_rate.toFixed(1)}% < 99.9%`);
|
|
2553
|
+
console.error(`[pipe] Rolling back...`);
|
|
2554
|
+
try {
|
|
2555
|
+
run(`git revert HEAD --no-edit && git push`, { cwd: repo, stdio: 'pipe' });
|
|
2556
|
+
run(`npx wrangler deploy`, { cwd: repo, stdio: 'pipe' });
|
|
2557
|
+
}
|
|
2558
|
+
catch { }
|
|
2559
|
+
status.publish = { passed: false, timestamp: new Date().toISOString() };
|
|
2560
|
+
savePipeStatus(status);
|
|
2561
|
+
process.exit(1);
|
|
2562
|
+
}
|
|
2563
|
+
status.publish = { passed: true, timestamp: new Date().toISOString(), bench: { concurrent: 500, success_rate: bench.success_rate, p50: bench.p50, p95: bench.p95, p99: bench.p99 } };
|
|
2363
2564
|
}
|
|
2364
|
-
catch {
|
|
2565
|
+
catch (e) {
|
|
2566
|
+
console.log(`[pipe] Skipping production bench (no API key or network error).`);
|
|
2567
|
+
status.publish = { passed: true, timestamp: new Date().toISOString() };
|
|
2568
|
+
}
|
|
2569
|
+
savePipeStatus(status);
|
|
2365
2570
|
console.log(`\n[pipe] Gate 4 PASSED ✓`);
|
|
2366
2571
|
console.log(`[pipe] ${sourceName} is LIVE`);
|
|
2367
2572
|
console.log(` API: https://api.latinfo.dev/${sourceName.replace(/-/g, '/')}/`);
|
|
2368
2573
|
console.log(` CLI: latinfo ${sourceName.replace(/-/g, ' ')}`);
|
|
2369
|
-
status.publish = { passed: true, timestamp: new Date().toISOString() };
|
|
2370
|
-
savePipeStatus(status);
|
|
2371
2574
|
}
|
|
2372
2575
|
async function pipeStatus(args) {
|
|
2373
2576
|
const [sourceName] = args;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "latinfo",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0",
|
|
4
4
|
"description": "Tax registry & procurement API for Latin America. Query RUC, DNI, NIT, licitaciones from Peru & Colombia. Offline MPHF search, full OCDS data, updated daily.",
|
|
5
5
|
"homepage": "https://latinfo.dev",
|
|
6
6
|
"repository": {
|