latinfo 0.8.0 → 0.9.0

package/dist/client-search.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Client-side search: server sends minimal posting list bytes,
+ * client does intersection + scoring, then fetches only the winning records.
+ *
+ * Flow:
+ *   1. GET /posting?q=... → compact IDs (8 bytes each, ~10-80KB total)
+ *   2. Client intersects + ranks by IDF locally
+ *   3. GET /records?ids=... → display data for top-20 (~5-10KB)
+ *   Total: 2 requests, ~20-90KB transferred
+ */
+interface ClientSearchOpts {
+    apiUrl: string;
+    apiKey: string;
+    country: string;
+    query: string;
+    json: boolean;
+    simLatency: number;
+    simBandwidth: number;
+}
+export declare function clientSearch(opts: ClientSearchOpts): Promise<void>;
+export {};

package/dist/client-search.js

@@ -0,0 +1,141 @@
+"use strict";
+/**
+ * Client-side search: server sends minimal posting list bytes,
+ * client does intersection + scoring, then fetches only the winning records.
+ *
+ * Flow:
+ *   1. GET /posting?q=... → compact IDs (8 bytes each, ~10-80KB total)
+ *   2. Client intersects + ranks by IDF locally
+ *   3. GET /records?ids=... → display data for top-20 (~5-10KB)
+ *   Total: 2 requests, ~20-90KB transferred
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.clientSearch = clientSearch;
+// --- Intersection on BigUint64Arrays ---
+function decodeIds(b64) {
+    const binary = atob(b64);
+    const buf = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++)
+        buf[i] = binary.charCodeAt(i);
+    const view = new DataView(buf.buffer);
+    const ids = [];
+    for (let i = 0; i < buf.length; i += 8)
+        ids.push(view.getBigUint64(i, true));
+    return ids;
+}
+function intersect(lists, limit) {
+    if (lists.length === 0)
+        return [];
+    if (lists.length === 1)
+        return lists[0].slice(0, limit);
+    lists.sort((a, b) => a.length - b.length);
+    let current = lists[0];
+    for (let li = 1; li < lists.length; li++) {
+        const other = lists[li];
+        const next = [];
+        let oi = 0;
+        for (let ci = 0; ci < current.length && next.length < limit; ci++) {
+            while (oi < other.length && other[oi] < current[ci])
+                oi++;
+            if (oi < other.length && other[oi] === current[ci])
+                next.push(current[ci]);
+        }
+        current = next;
+        if (current.length === 0)
+            break;
+    }
+    return current.slice(0, limit);
+}
+// --- IDF scoring (no record data needed) ---
+function scoreByIdf(dfs, totalDocs) {
+    let s = 0;
+    for (const df of dfs) {
+        if (df > 0)
+            s += Math.log(totalDocs / df);
+    }
+    return s;
+}
+async function simulateDelay(bytes, simLatency, simBandwidth) {
+    let delay = simLatency;
+    if (simBandwidth > 0)
+        delay += (bytes / 1024) / simBandwidth * 1000;
+    if (delay > 0)
+        await new Promise(r => setTimeout(r, delay));
+}
+async function clientSearch(opts) {
+    const { apiUrl, apiKey, country, query, json, simLatency, simBandwidth } = opts;
+    const headers = { Authorization: `Bearer ${apiKey}` };
+    // --- Request 1: posting lists (compact IDs) ---
+    const t0 = performance.now();
+    const postingRes = await fetch(`${apiUrl}/${country}/posting?q=${encodeURIComponent(query)}&limit=50000`, { headers });
+    const postingBody = await postingRes.text();
+    const postingBytes = new TextEncoder().encode(postingBody).length;
+    if (!postingRes.ok) {
+        const err = JSON.parse(postingBody);
+        console.error(err.message || err.error);
+        process.exit(1);
+    }
+    await simulateDelay(postingBytes, simLatency, simBandwidth);
+    const tPosting = performance.now();
+    // --- Client computation: decode + intersect ---
+    const tCompute0 = performance.now();
+    const data = JSON.parse(postingBody);
+    const tokenPostings = [];
+    let totalEntries = 0;
+    for (const t of data.terms) {
+        const ids = decodeIds(t.data);
+        tokenPostings.push(ids);
+        totalEntries += ids.length;
+    }
+    const candidateIds = intersect(tokenPostings, 20);
+    const tCompute1 = performance.now();
+    if (candidateIds.length === 0) {
+        if (json)
+            console.log(JSON.stringify({ results: [], diagnostics: {} }));
+        else
+            console.log('No results found.');
+        return;
+    }
+    // --- Request 2: fetch records for winning IDs ---
+    const idStrings = candidateIds.map(id => id.toString().padStart(10, '0'));
+    const tRecords0 = performance.now();
+    const recordsRes = await fetch(`${apiUrl}/${country}/records?ids=${idStrings.join(',')}`, { headers });
+    const recordsBody = await recordsRes.text();
+    const recordsBytes = new TextEncoder().encode(recordsBody).length;
+    await simulateDelay(recordsBytes, simLatency, simBandwidth);
+    const tRecords1 = performance.now();
+    const records = JSON.parse(recordsBody);
+    const tTotal = performance.now();
+    // --- Display ---
+    if (json) {
+        console.log(JSON.stringify({
+            results: records,
+            diagnostics: {
+                postingMs: Math.round(tPosting - t0),
+                postingBytes,
+                computeMs: Math.round(tCompute1 - tCompute0),
+                recordsMs: Math.round(tRecords1 - tRecords0),
+                recordsBytes,
+                totalMs: Math.round(tTotal - t0),
+                totalBytes: postingBytes + recordsBytes,
+                terms: data.terms.length,
+                entries: totalEntries,
+                candidates: candidateIds.length,
+            },
+        }));
+        return;
+    }
+    for (const r of records) {
+        const vals = Object.values(r);
+        console.log(`  ${vals[0]}  ${vals[1]}  [${vals[2] || ''}]`);
+    }
+    console.log(`\n${records.length} result(s)`);
+    console.log();
+    console.log(`  Request 1 (posting lists):  ${Math.round(tPosting - t0)}ms  ${(postingBytes / 1024).toFixed(1)} KB  (${totalEntries} IDs)`);
+    console.log(`  Client compute:             ${Math.round(tCompute1 - tCompute0)}ms  (decode + intersect)`);
+    console.log(`  Request 2 (records):        ${Math.round(tRecords1 - tRecords0)}ms  ${(recordsBytes / 1024).toFixed(1)} KB  (${records.length} records)`);
+    console.log(`  Total:                      ${Math.round(tTotal - t0)}ms  ${((postingBytes + recordsBytes) / 1024).toFixed(1)} KB transferred`);
+    if (simLatency > 0 || simBandwidth > 0) {
+        console.log(`  Simulation:                 +${simLatency}ms latency, ${simBandwidth || '∞'} KB/s`);
+    }
+}