@graphext/cuery 0.7.0 → 0.8.2

package/esm/src/apis/hasdata/serp.js

@@ -0,0 +1,133 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+import { mapParallel } from '../../helpers/async.js';
+import { fetchHasDataWithRetry, HASDATA_CONCURRENCY, parseAIO } from './helpers.js';
+const SEARCH_TYPE_TO_TBM = {
+    images: 'isch',
+    videos: 'vid',
+    news: 'nws',
+    shopping: 'shop',
+    local: 'lcl'
+};
+const SERP_ENDPOINT = 'https://api.hasdata.com/scrape/google/serp';
+function appendParam(url, key, value) {
+    url.searchParams.set(key, String(value));
+}
+function appendOptionalParam(url, key, value) {
+    if (value === undefined || value === null) {
+        return;
+    }
+    appendParam(url, key, value);
+}
+function formatBooleanParam(value, trueToken, falseToken) {
+    if (typeof value === 'boolean') {
+        return value ? trueToken : falseToken;
+    }
+    return String(value);
+}
+function normalizeLocale(value) {
+    return value ? value.toLowerCase() : undefined;
+}
+function normalizeDevice(device) {
+    if (!device) {
+        return undefined;
+    }
+    const normalized = device.toLowerCase();
+    if (normalized === 'desktop' || normalized === 'mobile' || normalized === 'tablet') {
+        return normalized;
+    }
+    return undefined;
+}
+function normalizeTbs(value) {
+    if (!value) {
+        return undefined;
+    }
+    if (Array.isArray(value)) {
+        return value.map(entry => entry.trim()).filter(Boolean).join(',');
+    }
+    return value;
+}
+function normalizeTbm(searchType) {
+    if (!searchType || searchType === 'all') {
+        return undefined;
+    }
+    const key = searchType.toLowerCase();
+    return SEARCH_TYPE_TO_TBM[key] || searchType;
+}
+function applySerpParams(url, options) {
+    appendOptionalParam(url, 'location', options.location);
+    appendOptionalParam(url, 'gl', normalizeLocale(options.country));
+    appendOptionalParam(url, 'hl', normalizeLocale(options.language));
+    appendOptionalParam(url, 'lr', options.contentLanguage);
+    appendOptionalParam(url, 'domain', options.domain);
+    appendOptionalParam(url, 'uule', options.encodedLocation);
+    const tbs = normalizeTbs(options.filters);
+    if (tbs) {
+        appendParam(url, 'tbs', tbs);
+    }
+    if (options.safeSearch !== undefined) {
+        const safeValue = typeof options.safeSearch === 'boolean'
+            ? options.safeSearch
+                ? 'active'
+                : 'off'
+            : options.safeSearch;
+        appendParam(url, 'safe', safeValue);
+    }
+    if (options.filterResults !== undefined) {
+        const filterValue = formatBooleanParam(options.filterResults, '1', '0');
+        appendParam(url, 'filter', filterValue);
+    }
+    if (options.preventAutoCorrect !== undefined) {
+        const nfprValue = formatBooleanParam(options.preventAutoCorrect, '1', '0');
+        appendParam(url, 'nfpr', nfprValue);
+    }
+    const tbm = normalizeTbm(options.type);
+    if (typeof options.offset === 'number' && options.offset >= 0) {
+        let start = Math.floor(options.offset);
+        if (tbm === 'lcl' && start % 20 !== 0) {
+            start = Math.floor(start / 20) * 20;
+        }
+        appendParam(url, 'start', start);
+    }
+    if (typeof options.resultsPerPage === 'number' && options.resultsPerPage > 0) {
+        const num = Math.min(Math.max(Math.floor(options.resultsPerPage), 10), 100);
+        appendParam(url, 'num', num);
+    }
+    if (tbm) {
+        appendParam(url, 'tbm', tbm);
+    }
+    const device = normalizeDevice(options.device);
+    if (device) {
+        appendParam(url, 'deviceType', device);
+    }
+    appendOptionalParam(url, 'ludocid', options.placeId);
+    appendOptionalParam(url, 'lsig', options.lsig);
+    appendOptionalParam(url, 'kgmid', options.entityId);
+    appendOptionalParam(url, 'si', options.searchId);
+}
+async function fetchSerpInternal(url) {
+    const response = await fetchHasDataWithRetry(url);
+    const content = (await response.json());
+    let aio = content.aiOverview;
+    if (aio && aio.pageToken && aio.hasdataLink) {
+        const aioResponse = await fetchHasDataWithRetry(aio.hasdataLink);
+        aio = await aioResponse.json();
+    }
+    if (aio) {
+        content.aiOverview = parseAIO(aio);
+    }
+    return content;
+}
+export async function fetchSerp(query, options) {
+    const url = new URL(SERP_ENDPOINT);
+    url.searchParams.set('q', query);
+    applySerpParams(url, options);
+    return fetchSerpInternal(url.toString());
+}
+export async function fetchSerpBatch(queries, options, maxConcurrency = HASDATA_CONCURRENCY) {
+    const url = new URL(SERP_ENDPOINT);
+    applySerpParams(url, options);
+    return mapParallel(queries, maxConcurrency, async (query) => {
+        url.searchParams.set('q', query);
+        return await fetchSerpInternal(url.toString());
+    });
+}

package/esm/src/schemas/index.d.ts

@@ -12,11 +12,15 @@
  */
 export type * from './persona.schema.js';
 export type * from './brand.schema.js';
+export type * from './entity.schema.js';
 export type * from './funnel.schema.js';
 export type * from './keyword.schema.js';
 export type * from './models.schema.js';
+export type * from './prompt.schema.js';
+export type * from './search.schema.js';
 export type * from './seedKeyword.schema.js';
-export type * from './topics.schema.js';
+export type * from './sentiment.schema.js';
+export type * from './sources.schema.js';
 export type * from './summary.schema.js';
-export type * from './prompt.schema.js';
+export type * from './topics.schema.js';
 //# sourceMappingURL=index.d.ts.map

package/esm/src/schemas/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,mBAAmB,CAAC;AACvC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,yBAAyB,CAAC;AAC7C,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,oBAAoB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAGH,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,mBAAmB,CAAC;AACvC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,oBAAoB,CAAC;AACxC,mBAAmB,yBAAyB,CAAC;AAC7C,mBAAmB,uBAAuB,CAAC;AAC3C,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,qBAAqB,CAAC;AACzC,mBAAmB,oBAAoB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graphext/cuery",
-  "version": "0.7.0",
+  "version": "0.8.2",
   "description": "Cuery tools for AI-powered keyword research and brand analysis",
   "repository": {
     "type": "git",

package/script/mod.d.ts

@@ -13,6 +13,8 @@ export * from './src/tools/personas.js';
 export * from './src/tools/search.js';
 export * from './src/tools/topics.js';
 export * from './src/helpers/utils.js';
+export * from './src/helpers/async.js';
+export * from './src/helpers/urls.js';
 export * from './src/tools/brands.js';
 export * from './src/tools/translate.js';
 export * from './src/tools/sentiment.js';
@@ -20,9 +22,10 @@ export * from './src/tools/summarize.js';
 export * from './src/tools/sources.js';
 export * from './src/tools/entities.js';
 export * from './src/tools/prompts.js';
+export * from './src/tools/scorer.js';
 export * from './src/helpers/seedKeywords.js';
 export * from './src/tools/generic.js';
-export * from './src/api.js';
+export * from './src/apis/hasdata/index.js';
 export * from './src/apis/chatgptScraper/index.js';
 export * from './src/apis/googleAds/keywordPlanner.js';
 export * from './src/schemas/index.js';

package/script/mod.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EACN,WAAW,EACX,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,SAAS,GACd,MAAM,0BAA0B,CAAC;AAGlC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,wBAAwB,CAAC;AACvC,cAAc,cAAc,CAAC;AAC7B,cAAc,oCAAoC,CAAC;AACnD,cAAc,wCAAwC,CAAC;AACvD,cAAc,wBAAwB,CAAC"}
+{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EACN,WAAW,EACX,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,SAAS,GACd,MAAM,0BAA0B,CAAC;AAGlC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,wBAAwB,CAAC;AACvC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oCAAoC,CAAC;AACnD,cAAc,wCAAwC,CAAC;AACvD,cAAc,wBAAwB,CAAC"}

package/script/mod.js

@@ -38,6 +38,8 @@ __exportStar(require("./src/tools/personas.js"), exports);
 __exportStar(require("./src/tools/search.js"), exports);
 __exportStar(require("./src/tools/topics.js"), exports);
 __exportStar(require("./src/helpers/utils.js"), exports);
+__exportStar(require("./src/helpers/async.js"), exports);
+__exportStar(require("./src/helpers/urls.js"), exports);
 __exportStar(require("./src/tools/brands.js"), exports);
 __exportStar(require("./src/tools/translate.js"), exports);
 __exportStar(require("./src/tools/sentiment.js"), exports);
@@ -45,9 +47,10 @@ __exportStar(require("./src/tools/summarize.js"), exports);
 __exportStar(require("./src/tools/sources.js"), exports);
 __exportStar(require("./src/tools/entities.js"), exports);
 __exportStar(require("./src/tools/prompts.js"), exports);
+__exportStar(require("./src/tools/scorer.js"), exports);
 __exportStar(require("./src/helpers/seedKeywords.js"), exports);
 __exportStar(require("./src/tools/generic.js"), exports);
-__exportStar(require("./src/api.js"), exports);
+__exportStar(require("./src/apis/hasdata/index.js"), exports);
 __exportStar(require("./src/apis/chatgptScraper/index.js"), exports);
 __exportStar(require("./src/apis/googleAds/keywordPlanner.js"), exports);
 __exportStar(require("./src/schemas/index.js"), exports);

package/script/src/apis/hasdata/index.d.ts

@@ -0,0 +1,6 @@
+export * from './helpers.js';
+export * from './aio.js';
+export * from './aim.js';
+export * from './serp.js';
+export * from './scrape.js';
+//# sourceMappingURL=index.d.ts.map

package/script/src/apis/hasdata/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC"}

package/script/src/apis/hasdata/index.js

@@ -0,0 +1,21 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./helpers.js"), exports);
+__exportStar(require("./aio.js"), exports);
+__exportStar(require("./aim.js"), exports);
+__exportStar(require("./serp.js"), exports);
+__exportStar(require("./scrape.js"), exports);

package/script/src/apis/hasdata/scrape.d.ts

@@ -0,0 +1,73 @@
+type ProxyType = 'datacenter' | 'residential';
+type OutputFormat = 'markdown' | 'text' | 'html';
+interface JSScenarioAction {
+    click?: string;
+    fill?: [string, string];
+    wait?: number;
+    waitFor?: string;
+    scroll?: string;
+    evaluate?: string;
+}
+export interface ScrapeOptions {
+    formats: Array<OutputFormat>;
+    proxyType?: ProxyType;
+    proxyCountry?: string;
+    extractLinks?: boolean;
+    wait?: number;
+    waitFor?: string;
+    blockResources?: boolean;
+    blockAds?: boolean;
+    blockUrls?: Array<string>;
+    jsRendering?: boolean;
+    jsScenario?: Array<JSScenarioAction>;
+    headers?: Record<string, string>;
+}
+export interface ScrapeResponse {
+    url?: string;
+    markdown?: string;
+    text?: string;
+    html?: string;
+    links?: Array<string>;
+}
+export interface BatchJobResponse {
+    jobId: string;
+    status: string;
+}
+export interface BatchJobStatus {
+    jobId: string;
+    status: string;
+    data: {
+        status: string;
+        requestsCount: number;
+        responsesCount: number;
+    };
+}
+/**
+ * In batch jobs, results are only links to json files containing the actual scrape results.
+*/
+export interface BatchResultItem {
+    query: Record<string, unknown>;
+    result: {
+        id: string;
+        status: string;
+        json?: string;
+    };
+}
+export interface BatchResults {
+    page: number;
+    limit: number;
+    total: number;
+    results: Array<BatchResultItem>;
+}
+export declare function scrapeWeb(url: string, options: ScrapeOptions): Promise<ScrapeResponse>;
+export declare function scrapeWebBatch(urls: Array<string>, options: ScrapeOptions, maxConcurrency?: number): Promise<Array<ScrapeResponse>>;
+/** Submit a batch scrape job to HasData API.
+ * IMPORTANT: results are not returned in original order! You need to match them by jobId and query.url.
+*/
+export declare function submitBatchScrapeJob(urls: Array<string>, options: ScrapeOptions): Promise<BatchJobResponse>;
+export declare function getBatchJobStatus(jobId: string): Promise<BatchJobStatus>;
+export declare function waitForBatchCompletion(jobId: string, pollInterval?: number, maxWaitTime?: number): Promise<BatchJobStatus>;
+export declare function getBatchJobPage(jobId: string, page?: number, limit?: number): Promise<BatchResults>;
+export declare function runBatchScrape(urls: Array<string>, options: ScrapeOptions, pageSize?: number, pollInterval?: number, maxWaitTime?: number): Promise<Array<ScrapeResponse>>;
+export {};
+//# sourceMappingURL=scrape.d.ts.map

package/script/src/apis/hasdata/scrape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/scrape.ts"],"names":[],"mappings":"AAiBA,KAAK,SAAS,GAAG,YAAY,GAAG,aAAa,CAAC;AAE9C,KAAK,YAAY,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;AAEjD,UAAU,gBAAgB;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC7B,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC7B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC9B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE;QACL,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,cAAc,EAAE,MAAM,CAAC;KACvB,CAAA;CACD;AAED;;EAEE;AAEF,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,EAAE;QACP,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,CAAC,EAAE,MAAM,CAAC;KACd,CAAC;CACF;AAED,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;CAChC;AAkJD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC,CAwC5F;AAED,wBAAsB,cAAc,CACnC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,EAAE,aAAa,EACtB,cAAc,GAAE,MAA4B,GAC1C,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAQhC;AAED;;EAEE;AACF,wBAAsB,oBAAoB,CACzC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,EAAE,aAAa,GACpB,OAAO,CAAC,gBAAgB,CAAC,CA8B3B;AAED,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAqB9E;AAED,wBAAsB,sBAAsB,CAC3C,KAAK,EAAE,MAAM,EACb,YAAY,GAAE,MAAa,EAC3B,WAAW,GAAE,MAAe,GAC1B,OAAO,CAAC,cAAc,CAAC,CAsBzB;AAED,wBAAsB,eAAe,CACpC,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,MAAU,EAChB,KAAK,GAAE,MAAY,GACjB,OAAO,CAAC,YAAY,CAAC,CAuBvB;AAED,wBAAsB,cAAc,CACnC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,EAAE,aAAa,EACtB,QAAQ,GAAE,MAAY,EACtB,YAAY,GAAE,MAAa,EAC3B,WAAW,GAAE,MAAe,GAC1B,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CA8DhC"}