@graphext/cuery 0.9.0 → 0.9.2

package/script/src/apis/hasdata/serp.js

@@ -0,0 +1,137 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchSerp = fetchSerp;
+exports.fetchSerpBatch = fetchSerpBatch;
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+const async_js_1 = require("../../helpers/async.js");
+const helpers_js_1 = require("./helpers.js");
+const SEARCH_TYPE_TO_TBM = {
+    images: 'isch',
+    videos: 'vid',
+    news: 'nws',
+    shopping: 'shop',
+    local: 'lcl'
+};
+const SERP_ENDPOINT = 'https://api.hasdata.com/scrape/google/serp';
+function appendParam(url, key, value) {
+    url.searchParams.set(key, String(value));
+}
+function appendOptionalParam(url, key, value) {
+    if (value === undefined || value === null) {
+        return;
+    }
+    appendParam(url, key, value);
+}
+function formatBooleanParam(value, trueToken, falseToken) {
+    if (typeof value === 'boolean') {
+        return value ? trueToken : falseToken;
+    }
+    return String(value);
+}
+function normalizeLocale(value) {
+    return value ? value.toLowerCase() : undefined;
+}
+function normalizeDevice(device) {
+    if (!device) {
+        return undefined;
+    }
+    const normalized = device.toLowerCase();
+    if (normalized === 'desktop' || normalized === 'mobile' || normalized === 'tablet') {
+        return normalized;
+    }
+    return undefined;
+}
+function normalizeTbs(value) {
+    if (!value) {
+        return undefined;
+    }
+    if (Array.isArray(value)) {
+        return value.map(entry => entry.trim()).filter(Boolean).join(',');
+    }
+    return value;
+}
+function normalizeTbm(searchType) {
+    if (!searchType || searchType === 'all') {
+        return undefined;
+    }
+    const key = searchType.toLowerCase();
+    return SEARCH_TYPE_TO_TBM[key] || searchType;
+}
+function applySerpParams(url, options) {
+    appendOptionalParam(url, 'location', options.location);
+    appendOptionalParam(url, 'gl', normalizeLocale(options.country));
+    appendOptionalParam(url, 'hl', normalizeLocale(options.language));
+    appendOptionalParam(url, 'lr', options.contentLanguage);
+    appendOptionalParam(url, 'domain', options.domain);
+    appendOptionalParam(url, 'uule', options.encodedLocation);
+    const tbs = normalizeTbs(options.filters);
+    if (tbs) {
+        appendParam(url, 'tbs', tbs);
+    }
+    if (options.safeSearch !== undefined) {
+        const safeValue = typeof options.safeSearch === 'boolean'
+            ? options.safeSearch
+                ? 'active'
+                : 'off'
+            : options.safeSearch;
+        appendParam(url, 'safe', safeValue);
+    }
+    if (options.filterResults !== undefined) {
+        const filterValue = formatBooleanParam(options.filterResults, '1', '0');
+        appendParam(url, 'filter', filterValue);
+    }
+    if (options.preventAutoCorrect !== undefined) {
+        const nfprValue = formatBooleanParam(options.preventAutoCorrect, '1', '0');
+        appendParam(url, 'nfpr', nfprValue);
+    }
+    const tbm = normalizeTbm(options.type);
+    if (typeof options.offset === 'number' && options.offset >= 0) {
+        let start = Math.floor(options.offset);
+        if (tbm === 'lcl' && start % 20 !== 0) {
+            start = Math.floor(start / 20) * 20;
+        }
+        appendParam(url, 'start', start);
+    }
+    if (typeof options.resultsPerPage === 'number' && options.resultsPerPage > 0) {
+        const num = Math.min(Math.max(Math.floor(options.resultsPerPage), 10), 100);
+        appendParam(url, 'num', num);
+    }
+    if (tbm) {
+        appendParam(url, 'tbm', tbm);
+    }
+    const device = normalizeDevice(options.device);
+    if (device) {
+        appendParam(url, 'deviceType', device);
+    }
+    appendOptionalParam(url, 'ludocid', options.placeId);
+    appendOptionalParam(url, 'lsig', options.lsig);
+    appendOptionalParam(url, 'kgmid', options.entityId);
+    appendOptionalParam(url, 'si', options.searchId);
+}
+async function fetchSerpInternal(url) {
+    const response = await (0, helpers_js_1.fetchHasDataWithRetry)(url);
+    const content = (await response.json());
+    let aio = content.aiOverview;
+    if (aio && aio.pageToken && aio.hasdataLink) {
+        const aioResponse = await (0, helpers_js_1.fetchHasDataWithRetry)(aio.hasdataLink);
+        aio = await aioResponse.json();
+    }
+    if (aio) {
+        content.aiOverview = (0, helpers_js_1.parseAIO)(aio);
+    }
+    return content;
+}
+async function fetchSerp(query, options) {
+    const url = new URL(SERP_ENDPOINT);
+    url.searchParams.set('q', query);
+    applySerpParams(url, options);
+    return fetchSerpInternal(url.toString());
+}
+async function fetchSerpBatch(queries, options, maxConcurrency = helpers_js_1.HASDATA_CONCURRENCY) {
+    const url = new URL(SERP_ENDPOINT);
+    applySerpParams(url, options);
+    return (0, async_js_1.mapParallel)(queries, maxConcurrency, async (query) => {
+        url.searchParams.set('q', query);
+        return await fetchSerpInternal(url.toString());
+    });
+}

package/script/src/schemas/index.d.ts

@@ -12,11 +12,15 @@
  */
 export type * from './persona.schema.js';
 export type * from './brand.schema.js';
+export type * from './entity.schema.js';
 export type * from './funnel.schema.js';
 export type * from './keyword.schema.js';
 export type * from './models.schema.js';
+export type * from './prompt.schema.js';
+export type * from './search.schema.js';
 export type * from './seedKeyword.schema.js';
-export type * from './topics.schema.js';
+export type * from './sentiment.schema.js';
+export type * from './sources.schema.js';
 export type * from './summary.schema.js';
-export type * from './prompt.schema.js';
+export type * from './topics.schema.js';
 //# sourceMappingURL=index.d.ts.map

package/script/src/schemas/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,mBAAmB,CAAC;AACvC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,yBAAyB,CAAC;AAC7C,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,oBAAoB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,mBAAmB,CAAC;AACvC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,yBAAyB,CAAC;AAC7C,mBAAmB,uBAAuB,CAAC;AAC3C,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,oBAAoB,CAAC"}